diff --git a/crates/heal/src/heal/mod.rs b/crates/heal/src/heal/mod.rs index 59d7d25bf..fff33d03d 100644 --- a/crates/heal/src/heal/mod.rs +++ b/crates/heal/src/heal/mod.rs @@ -19,6 +19,7 @@ pub mod manager; pub mod progress; pub mod resume; pub mod storage; +mod storage_compat; pub mod task; pub mod utils; diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index c6a92e09c..bdd6b6c8c 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -19,7 +19,6 @@ use rustfs_ecstore::{ disk::{DiskStore, endpoint::Endpoint}, error::StorageError, store::ECStore, - store_api::{ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions, PutObjReader as EcstorePutObjReader}, }; use rustfs_madmin::heal_commands::HealResultItem; use rustfs_storage_api::{ @@ -29,6 +28,8 @@ use rustfs_storage_api::{ use std::sync::Arc; use tracing::{debug, error, warn}; +pub use super::storage_compat::{HealObjectInfo, HealObjectOptions, HealPutObjReader}; + const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_STORAGE: &str = "storage"; const EVENT_HEAL_STORAGE_OBJECT_IO: &str = "heal_storage_object_io"; @@ -37,10 +38,6 @@ const EVENT_HEAL_STORAGE_OBJECT_VERIFY: &str = "heal_storage_object_verify"; const EVENT_HEAL_STORAGE_ADMIN_OP: &str = "heal_storage_admin_op"; const EVENT_HEAL_STORAGE_REPAIR_OP: &str = "heal_storage_repair_op"; -pub type HealObjectInfo = EcstoreObjectInfo; -pub type HealObjectOptions = EcstoreObjectOptions; -pub type HealPutObjReader = EcstorePutObjReader; - /// Disk status for heal operations #[derive(Debug, Clone, PartialEq, Eq)] pub enum DiskStatus { diff --git a/crates/heal/src/heal/storage_compat.rs b/crates/heal/src/heal/storage_compat.rs new file mode 100644 index 000000000..8154620e4 --- /dev/null +++ b/crates/heal/src/heal/storage_compat.rs @@ -0,0 +1,21 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_ecstore::store_api::{ + ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions, PutObjReader as EcstorePutObjReader, +}; + +pub type HealObjectInfo = EcstoreObjectInfo; +pub type HealObjectOptions = EcstoreObjectOptions; +pub type HealPutObjReader = EcstorePutObjReader; diff --git a/crates/heal/tests/heal_integration_test.rs b/crates/heal/tests/heal_integration_test.rs index 1fe5045d5..9250c3d4b 100644 --- a/crates/heal/tests/heal_integration_test.rs +++ b/crates/heal/tests/heal_integration_test.rs @@ -18,11 +18,10 @@ use rustfs_ecstore::{ disk::endpoint::Endpoint, endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, store::ECStore, - store_api::{ObjectOptions, PutObjReader}, }; use rustfs_heal::heal::{ manager::{HealConfig, HealManager}, - storage::{ECStoreHealStorage, HealStorageAPI}, + storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI}, task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType}, }; use rustfs_storage_api::{BucketOperations, ObjectIO as _, ObjectOperations as _}; diff --git a/crates/iam/src/store.rs b/crates/iam/src/store.rs index 9e189b99b..7092543f4 100644 --- a/crates/iam/src/store.rs +++ b/crates/iam/src/store.rs @@ -13,6 +13,7 @@ // limitations under the License. pub mod object; +mod storage_compat; use crate::cache::Cache; use crate::error::Result; diff --git a/crates/iam/src/store/object.rs b/crates/iam/src/store/object.rs index c28cc149e..ed6d936c7 100644 --- a/crates/iam/src/store/object.rs +++ b/crates/iam/src/store/object.rs @@ -29,7 +29,6 @@ use rustfs_ecstore::{ com::{delete_config, read_config_no_lock, read_config_with_metadata, save_config, save_config_with_opts}, }, store::ECStore, - store_api::{ObjectInfo, ObjectOptions}, }; use rustfs_io_metrics::record_system_path_failure; use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc}; @@ -43,6 +42,8 @@ use tokio::sync::mpsc::{self, Sender}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, warn}; +use super::storage_compat::{IamObjectInfo, IamObjectOptions}; + pub static IAM_CONFIG_PREFIX: LazyLock = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam")); pub static IAM_CONFIG_USERS_PREFIX: LazyLock = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam/users/")); pub static IAM_CONFIG_SERVICE_ACCOUNTS_PREFIX: LazyLock = @@ -60,7 +61,7 @@ pub static IAM_CONFIG_POLICY_DB_SERVICE_ACCOUNTS_PREFIX: LazyLock = pub static IAM_CONFIG_POLICY_DB_GROUPS_PREFIX: LazyLock = LazyLock::new(|| format!("{RUSTFS_CONFIG_PREFIX}/iam/policydb/groups/")); -type ObjectInfoOrErr = StorageObjectInfoOrErr; +type ObjectInfoOrErr = StorageObjectInfoOrErr; const IAM_IDENTITY_FILE: &str = "identity.json"; const IAM_POLICY_FILE: &str = "policy.json"; @@ -282,7 +283,7 @@ impl ObjectStore { entry.cooldown_until = Some(Instant::now() + IAM_LAZY_REWRITE_COOLDOWN); } - fn maybe_schedule_lazy_rewrite(&self, path: &str, outcome: &DecryptOutcome, object_info: &ObjectInfo) { + fn maybe_schedule_lazy_rewrite(&self, path: &str, outcome: &DecryptOutcome, object_info: &IamObjectInfo) { if !Self::should_lazy_rewrite(outcome.source) { return; } @@ -322,7 +323,7 @@ impl ObjectStore { async fn lazy_rewrite_iam_config(&self, path: &str, plain: &[u8], etag: &str) -> std::result::Result<(), StorageError> { let encrypted = Self::encrypt_data_with_master_key(plain).map_err(StorageError::other)?; - let mut opts = ObjectOptions { + let mut opts = IamObjectOptions { max_parity: true, ..Default::default() }; @@ -343,9 +344,9 @@ impl ObjectStore { Self::prepare_data_for_storage(data) } - async fn load_iamconfig_bytes_with_metadata(&self, path: impl AsRef + Send) -> Result<(Vec, ObjectInfo)> { + async fn load_iamconfig_bytes_with_metadata(&self, path: impl AsRef + Send) -> Result<(Vec, IamObjectInfo)> { let path_ref = path.as_ref(); - let (data, obj) = read_config_with_metadata(self.object_api.clone(), path_ref, &ObjectOptions::default()).await?; + let (data, obj) = read_config_with_metadata(self.object_api.clone(), path_ref, &IamObjectOptions::default()).await?; let outcome = Self::decrypt_data_with_source(&data)?; self.maybe_schedule_lazy_rewrite(path_ref, &outcome, &obj); @@ -640,7 +641,7 @@ impl Store for ObjectStore { } async fn load_iam_config(&self, path: impl AsRef + Send) -> Result { let path_ref = path.as_ref(); - let (data, obj) = read_config_with_metadata(self.object_api.clone(), path_ref, &ObjectOptions::default()).await?; + let (data, obj) = read_config_with_metadata(self.object_api.clone(), path_ref, &IamObjectOptions::default()).await?; let outcome = match Self::decrypt_data_with_source(&data) { Ok(v) => v, diff --git a/crates/iam/src/store/storage_compat.rs b/crates/iam/src/store/storage_compat.rs new file mode 100644 index 000000000..45eaa9d95 --- /dev/null +++ b/crates/iam/src/store/storage_compat.rs @@ -0,0 +1,18 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_ecstore::store_api::{ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions}; + +pub(super) type IamObjectInfo = EcstoreObjectInfo; +pub(super) type IamObjectOptions = EcstoreObjectOptions; diff --git a/crates/notify/src/event.rs b/crates/notify/src/event.rs index 527487c97..df35e9b28 100644 --- a/crates/notify/src/event.rs +++ b/crates/notify/src/event.rs @@ -19,7 +19,7 @@ use rustfs_s3_types::{EventName, event_schema_version}; use serde::{Deserialize, Serialize}; use url::form_urlencoded; -pub type NotifyObjectInfo = rustfs_ecstore::store_api::ObjectInfo; +use crate::storage_compat::NotifyObjectInfo; /// Represents the identity of the user who triggered the event #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/crates/notify/src/lib.rs b/crates/notify/src/lib.rs index adb72e4a4..9b83662ce 100644 --- a/crates/notify/src/lib.rs +++ b/crates/notify/src/lib.rs @@ -36,11 +36,12 @@ mod runtime_facade; mod runtime_view; mod services; mod status_view; +mod storage_compat; pub use bucket_config_manager::NotifyBucketConfigManager; pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem}; pub use error::{LifecycleError, NotificationError}; -pub use event::{Event, EventArgs, EventArgsBuilder, NotifyObjectInfo}; +pub use event::{Event, EventArgs, EventArgsBuilder}; pub use event_bridge::{LiveEventHistory, NotifyEventBridge}; pub use global::{ initialize, initialize_live_events, is_notification_system_initialized, notification_metrics_snapshot, notification_system, @@ -54,3 +55,4 @@ pub use runtime_facade::NotifyRuntimeFacade; pub use runtime_view::NotifyRuntimeView; pub use services::NotifyServices; pub use status_view::NotifyStatusView; +pub use storage_compat::NotifyObjectInfo; diff --git a/crates/notify/src/storage_compat.rs b/crates/notify/src/storage_compat.rs new file mode 100644 index 000000000..6e21c2df4 --- /dev/null +++ b/crates/notify/src/storage_compat.rs @@ -0,0 +1,15 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub type NotifyObjectInfo = rustfs_ecstore::store_api::ObjectInfo; diff --git a/crates/protocols/src/swift/mod.rs b/crates/protocols/src/swift/mod.rs index bfdb06407..4fb818843 100644 --- a/crates/protocols/src/swift/mod.rs +++ b/crates/protocols/src/swift/mod.rs @@ -51,6 +51,7 @@ pub mod ratelimit; pub mod router; pub mod slo; pub mod staticweb; +pub mod storage_compat; pub mod symlink; pub mod sync; pub mod tempurl; diff --git a/crates/protocols/src/swift/object.rs b/crates/protocols/src/swift/object.rs index 3224391f5..1acb5fa61 100644 --- a/crates/protocols/src/swift/object.rs +++ b/crates/protocols/src/swift/object.rs @@ -55,25 +55,18 @@ use super::{SwiftError, SwiftResult}; use axum::http::HeaderMap; use rustfs_credentials::Credentials; use rustfs_ecstore::resolve_object_store_handle; -use rustfs_ecstore::store_api::{ - GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions, - PutObjReader as EcstorePutObjReader, -}; use rustfs_rio::HashReader; use rustfs_storage_api::{BucketOperations, BucketOptions, ObjectIO as _, ObjectOperations as _}; use std::collections::HashMap; use tracing::debug; use tracing::error; +pub use super::storage_compat::{SwiftGetObjectReader, SwiftObjectInfo, SwiftObjectOptions, SwiftPutObjReader}; + const LOG_COMPONENT_PROTOCOLS: &str = "protocols"; const LOG_SUBSYSTEM_SWIFT_OBJECT: &str = "swift_object"; const EVENT_SWIFT_OBJECT_STORAGE_STATE: &str = "swift_object_storage_state"; -pub type SwiftGetObjectReader = EcstoreGetObjectReader; -pub type SwiftObjectInfo = EcstoreObjectInfo; -pub type SwiftObjectOptions = EcstoreObjectOptions; -pub type SwiftPutObjReader = EcstorePutObjReader; - /// Maximum number of metadata headers allowed per object (Swift standard) const MAX_METADATA_COUNT: usize = 90; diff --git a/crates/protocols/src/swift/storage_compat.rs b/crates/protocols/src/swift/storage_compat.rs new file mode 100644 index 000000000..d9effb419 --- /dev/null +++ b/crates/protocols/src/swift/storage_compat.rs @@ -0,0 +1,23 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_ecstore::store_api::{ + GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions, + PutObjReader as EcstorePutObjReader, +}; + +pub type SwiftGetObjectReader = EcstoreGetObjectReader; +pub type SwiftObjectInfo = EcstoreObjectInfo; +pub type SwiftObjectOptions = EcstoreObjectOptions; +pub type SwiftPutObjReader = EcstorePutObjReader; diff --git a/crates/s3select-api/src/lib.rs b/crates/s3select-api/src/lib.rs index 56688ea91..be937e73f 100644 --- a/crates/s3select-api/src/lib.rs +++ b/crates/s3select-api/src/lib.rs @@ -19,6 +19,7 @@ use std::fmt::Display; pub mod object_store; pub mod query; pub mod server; +mod storage_compat; #[cfg(test)] mod test; diff --git a/crates/s3select-api/src/object_store.rs b/crates/s3select-api/src/object_store.rs index f01836475..b170fa1b0 100644 --- a/crates/s3select-api/src/object_store.rs +++ b/crates/s3select-api/src/object_store.rs @@ -29,9 +29,6 @@ use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object use rustfs_ecstore::resolve_object_store_handle; use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; use rustfs_ecstore::store::ECStore; -use rustfs_ecstore::store_api::{ - GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions, -}; use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _}; use s3s::S3Result; use s3s::dto::SelectObjectContentInput; @@ -51,6 +48,8 @@ use tokio::io::{AsyncRead, ReadBuf}; use tokio_util::io::ReaderStream; use transform_stream::AsyncTryStream; +use crate::storage_compat::{SelectGetObjectReader, SelectObjectInfo, SelectObjectOptions}; + /// Maximum allowed object size for JSON DOCUMENT mode. /// /// JSON DOCUMENT format requires loading the entire file into memory for DOM @@ -68,10 +67,6 @@ pub const MAX_JSON_DOCUMENT_BYTES: u64 = 128 * 1024 * 1024; pub const INVALID_SCAN_RANGE_MESSAGE: &str = "The value of a parameter in ScanRange element is invalid. Check the service API documentation and try again."; -type SelectGetObjectReader = EcstoreGetObjectReader; -type SelectObjectInfo = EcstoreObjectInfo; -type SelectObjectOptions = EcstoreObjectOptions; - #[derive(Debug)] pub struct EcObjectStore { input: Arc, diff --git a/crates/s3select-api/src/storage_compat.rs b/crates/s3select-api/src/storage_compat.rs new file mode 100644 index 000000000..ded8394b4 --- /dev/null +++ b/crates/s3select-api/src/storage_compat.rs @@ -0,0 +1,21 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_ecstore::store_api::{ + GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions, +}; + +pub(crate) type SelectGetObjectReader = EcstoreGetObjectReader; +pub(crate) type SelectObjectInfo = EcstoreObjectInfo; +pub(crate) type SelectObjectOptions = EcstoreObjectOptions; diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index f1ddccfd3..84fdd856c 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -33,13 +33,16 @@ use rustfs_ecstore::{ config::{com::save_config, storageclass}, disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, error::{Error, Result as StorageResult, StorageError}, - store_api::{GetObjectReader, ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader}, }; -use rustfs_storage_api::{HTTPRangeSpec, ObjectIO}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use tokio::time::{Duration, Instant, sleep, timeout}; use tracing::warn; +pub use crate::storage_compat::{ + ScannerGetObjectReader, ScannerObjectIO, ScannerObjectInfo, ScannerObjectOptions, ScannerObjectToDelete, ScannerPutObjReader, +}; +use crate::storage_compat::{ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions}; + // Data usage constants pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; @@ -61,38 +64,6 @@ const EVENT_SCANNER_CACHE_LOAD_STATE: &str = "scanner_cache_load_state"; const EVENT_SCANNER_CACHE_SAVE_STATE: &str = "scanner_cache_save_state"; static CACHE_SAVE_METRICS_ONCE: Once = Once::new(); -pub type ScannerGetObjectReader = GetObjectReader; -pub type ScannerObjectInfo = ObjectInfo; -pub type ScannerObjectOptions = ObjectOptions; -pub type ScannerObjectToDelete = ObjectToDelete; -pub type ScannerPutObjReader = PutObjReader; - -pub trait ScannerObjectIO: - ObjectIO< - Error = Error, - RangeSpec = HTTPRangeSpec, - HeaderMap = HeaderMap, - ObjectOptions = ObjectOptions, - ObjectInfo = ObjectInfo, - GetObjectReader = GetObjectReader, - PutObjectReader = PutObjReader, - > -{ -} - -impl ScannerObjectIO for T where - T: ObjectIO< - Error = Error, - RangeSpec = HTTPRangeSpec, - HeaderMap = HeaderMap, - ObjectOptions = ObjectOptions, - ObjectInfo = ObjectInfo, - GetObjectReader = GetObjectReader, - PutObjectReader = PutObjReader, - > -{ -} - pub const DATA_USAGE_SCAN_CHECKPOINT_VERSION: u16 = 1; // Data usage paths (computed at runtime) diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index 614118f25..b71bed5d4 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -28,6 +28,7 @@ pub mod scanner_budget; pub mod scanner_folder; pub mod scanner_io; pub mod sleeper; +mod storage_compat; pub use data_usage_define::*; pub use error::ScannerError; diff --git a/crates/scanner/src/storage_compat.rs b/crates/scanner/src/storage_compat.rs new file mode 100644 index 000000000..977a294a9 --- /dev/null +++ b/crates/scanner/src/storage_compat.rs @@ -0,0 +1,55 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use http::HeaderMap; +use rustfs_ecstore::{ + error::Error, + store_api::{ + GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions, + ObjectToDelete as EcstoreObjectToDelete, PutObjReader as EcstorePutObjReader, + }, +}; +use rustfs_storage_api::{HTTPRangeSpec, ObjectIO}; + +pub type ScannerGetObjectReader = EcstoreGetObjectReader; +pub type ScannerObjectInfo = EcstoreObjectInfo; +pub type ScannerObjectOptions = EcstoreObjectOptions; +pub type ScannerObjectToDelete = EcstoreObjectToDelete; +pub type ScannerPutObjReader = EcstorePutObjReader; + +pub trait ScannerObjectIO: + ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ScannerObjectOptions, + ObjectInfo = ScannerObjectInfo, + GetObjectReader = ScannerGetObjectReader, + PutObjectReader = ScannerPutObjReader, + > +{ +} + +impl ScannerObjectIO for T where + T: ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ScannerObjectOptions, + ObjectInfo = ScannerObjectInfo, + GetObjectReader = ScannerGetObjectReader, + PutObjectReader = ScannerPutObjReader, + > +{ +} diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index d81a84188..262af2996 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -800,6 +800,30 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block diff hygiene, direct import scan, Rust risk scan, full pre-commit, and required three-expert review passed. +- [x] `API-027` Clean remaining external storage DTO imports. + - Current branch: `overtrue/arch-storage-compat-contract-cleanup`. + - Completed slice: move table catalog, IAM object-store, admin zip-download, + capacity dirty-scope tests, heal integration tests, scanner, Swift, S3 + Select, and notify event payloads from raw ECStore `store_api` DTO imports + to crate-local compatibility aliases/modules. + - Acceptance: non-ECStore direct `rustfs_ecstore::store_api` references are + limited to explicit boundary alias points in RustFS storage plus scanner, + heal, IAM, notify, Swift, and S3 Select compatibility modules; table + catalog, affected tests, and protocol/scanner/notification consumers + consume those boundary names instead of raw ECStore DTO paths. + - Must preserve: table catalog storage trait bindings, IAM metadata/lazy + rewrite behavior, object zip preflight/read semantics, capacity dirty-disk + assertions, heal integration object read/write behavior, scanner cache + load/save semantics, Swift object read/write/copy/delete behavior, S3 + Select object-store reads, notify event payload shape, and ECStore-owned DTO + concrete shapes. + - Risk defense: this slice changes import ownership and type aliases only; it + does not move DTO definitions, alter serialization, change object-store + implementation bodies, or adjust runtime control flow. + - Verification: focused compile/tests, migration/layer guards, formatting, + diff hygiene, direct import scan, Rust risk scan, full pre-commit, and + required three-expert review passed. + ## Phase 8 Background Controller Tasks - [x] `BGC-001` Inventory background services. @@ -1076,70 +1100,54 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | Scanner, heal, notify, Swift, S3 Select, and RustFS storage/app consumers now use crate-local DTO aliases; the erasure CI fix is isolated to zero-length shard reconstruction. | -| Migration preservation | passed | ECStore remains the owner of object DTOs/readers/walk filters/implementation behavior; aliases preserve concrete types and non-empty erasure encode/decode paths still use the existing encoders. | -| Testing/verification | passed | Focused DTO/erasure compile/tests, migration/layer guards, direct import scan, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` passed. | +| Quality/architecture | passed | Table catalog, IAM, admin zip, capacity, heal integration, scanner, Swift, S3 Select, and notify consumers now use local DTO compatibility aliases instead of raw ECStore `store_api` imports. | +| Migration preservation | passed | ECStore remains the owner of object DTO/readers; alias boundaries preserve concrete associated types for storage traits, IAM config reads, zip preflight reads, scanner cache I/O, Swift/S3 Select reads, notify payloads, capacity assertions, and heal integration writes. | +| Testing/verification | passed | Focused compile/tests, migration/layer guards, direct import scan, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` passed. | ## Verification Notes Passed before push: -- `cargo check --tests -p rustfs -p rustfs-scanner -p rustfs-heal -p - rustfs-protocols -p rustfs-notify -p rustfs-s3select-api`: passed. -- `cargo test -p rustfs-scanner -p rustfs-heal -p rustfs-notify -p - rustfs-protocols -p rustfs-s3select-api`: passed. -- `cargo test -p rustfs --lib app::object_usecase::tests`: passed; 85 passed, - 2 ignored. -- `cargo test -p rustfs --lib app::bucket_usecase::tests`: passed; 64 passed. -- `cargo test -p rustfs --lib app::multipart_usecase::tests`: passed; 24 - passed, 4 ignored. -- `cargo test -p rustfs --lib storage::s3_api::bucket::tests`: passed; 22 +- `cargo check --tests -p rustfs -p rustfs-iam -p rustfs-heal`: passed. +- `cargo check --tests -p rustfs-iam -p rustfs-heal -p rustfs-scanner -p + rustfs-protocols -p rustfs-s3select-api -p rustfs-notify`: passed. +- `cargo test -p rustfs --lib table_catalog_store_trait`: passed; 2 passed. +- `cargo test -p rustfs --lib admin::handlers::object_zip_download::tests`: + passed; 28 passed. +- `cargo test -p rustfs --lib app::capacity_dirty_scope_test`: passed; 2 passed. -- `cargo test -p rustfs --lib storage::ecfs_test::tests`: passed; 59 passed, - 14 ignored. -- `rg -n 'rustfs_ecstore::store_api' rustfs/src crates --glob +- `cargo test -p rustfs-iam store::object::tests`: passed; 8 passed. +- `cargo test -p rustfs-heal --test heal_integration_test`: passed; 5 passed. +- `cargo check --tests -p rustfs-scanner -p rustfs-protocols -p + rustfs-s3select-api -p rustfs-notify`: passed. +- `cargo test -p rustfs-scanner -p rustfs-protocols -p rustfs-s3select-api -p + rustfs-notify`: passed; notify 82 passed, protocols 13 passed, S3 Select 49 + passed, scanner 151 passed plus lifecycle integration 1 passed and 14 + ignored. +- `rg -n 'rustfs_ecstore::store_api|store_api::\{' rustfs/src crates --glob '!crates/ecstore/**' --glob '*.rs'`: remaining matches are deliberate - boundary alias definitions. -- `cargo test -p rustfs-ecstore erasure_coding::erasure::tests`: passed; 31 - passed. -- `PROPTEST_CASES=1024 cargo test -p rustfs-ecstore - erasure_coding::erasure::tests::decode_data_and_parity_round_trips_bounded_recoverability`: - passed. -- `cargo test -p rustfs-ecstore - rpc::peer_s3_client::tests::local_get_bucket_info_survives_prior_walk_timeout`: - passed after a full-crate `cargo test -p rustfs-ecstore` run exposed this - unrelated ordinary-harness global-state interference. + boundary alias definitions in RustFS storage and crate-local compat modules. - `./scripts/check_architecture_migration_rules.sh`: passed. - `./scripts/check_layer_dependencies.sh`: passed. - `cargo fmt --all --check`: passed. - `git diff --check`: passed. -- Rust risk scan: new hits are crate-local alias casts plus the empty-erasure - test expectations; no new production `unwrap`/`expect`, panic/todo markers, - `unsafe`, process-spawning calls, println/eprintln, or relaxed ordering in - added Rust lines. -- `make pre-commit`: passed; nextest reported 6218 tests passed and 111 - skipped, and doctests passed. +- Rust risk scan: new hits are crate-local alias casts only; no new + `unwrap`/`expect`, panic/todo markers, `unsafe`, process-spawning calls, + println/eprintln, or relaxed ordering in added Rust lines. +- `make pre-commit`: passed. Notes: -- This slice follows the external operation consumer cleanup and keeps the old - aggregate facade, DTO/helper, response, and operation contract guards active. -- External DTO consumers now name ECStore-owned object DTOs through local - semantic aliases in their owning crates or RustFS storage boundary module. -- The slice does not alter object metadata shape, options defaults, - reader/writer behavior, delete replication DTO handling, scanner cache - semantics, heal storage metadata semantics, Swift/S3 Select object reads, - notification event payloads, S3 response DTO mapping, or runtime storage - behavior. -- The CI follow-up preserves empty-object erasure reconstruction by filling - missing zero-length shards before calling encoders that reject zero-byte SIMD - shard sizes. +- This slice is stacked on API-026 while `rustfs/rustfs#3566` is pending. +- Direct ECStore `store_api` references outside ECStore now remain only at + explicit alias boundary points. +- The slice does not alter table catalog storage behavior, IAM config storage, + admin zip download object reads, capacity dirty-disk behavior, heal + integration semantics, scanner cache I/O, Swift/S3 Select object reads, + notification event payloads, or ECStore DTO definitions. ## Handoff Notes -- External DTO consumer cleanup is based on `main` after - `rustfs/rustfs#3565` merged and carries the empty-object erasure recovery - CI follow-up because the merged PR did not include that post-merge fix. - Continue with larger consumer-migration batches; keep ECStore-owned DTOs/readers/walk filters in ECStore until their concrete behavior is isolated enough for a pure-move slice. diff --git a/rustfs/src/admin/handlers/object_zip_download.rs b/rustfs/src/admin/handlers/object_zip_download.rs index 01f6fca86..fc1b82e1b 100644 --- a/rustfs/src/admin/handlers/object_zip_download.rs +++ b/rustfs/src/admin/handlers/object_zip_download.rs @@ -31,7 +31,7 @@ use matchit::Params; use rand::RngExt; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_credentials::get_global_action_cred; -use rustfs_ecstore::{global::get_global_region, store_api::ObjectOptions}; +use rustfs_ecstore::global::get_global_region; use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_storage_api::{BucketOperations, ListOperations as _, ObjectIO as _, ObjectOperations as _, bucket::BucketOptions}; use rustfs_trusted_proxies::{ClientInfo, ValidationMode}; @@ -49,6 +49,8 @@ use tokio_util::io::ReaderStream; use url::form_urlencoded; use uuid::Uuid; +use crate::storage::StorageObjectOptions as ObjectOptions; + const OBJECT_ZIP_DOWNLOAD_TOKEN_TTL: Duration = Duration::minutes(5); const ZIP_STREAM_BUFFER_SIZE: usize = 1024 * 1024; const ZIP_OBJECT_BUFFER_SIZE: usize = 128 * 1024; diff --git a/rustfs/src/app/capacity_dirty_scope_test.rs b/rustfs/src/app/capacity_dirty_scope_test.rs index de948d612..54d2b0a2c 100644 --- a/rustfs/src/app/capacity_dirty_scope_test.rs +++ b/rustfs/src/app/capacity_dirty_scope_test.rs @@ -18,7 +18,6 @@ use rustfs_ecstore::{ disk::endpoint::Endpoint, endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, store::ECStore, - store_api::{ObjectOptions, PutObjReader}, }; use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager}; use rustfs_storage_api::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _}; @@ -35,6 +34,8 @@ use tokio::fs; use tokio_util::sync::CancellationToken; use uuid::Uuid; +use crate::storage::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader}; + static CAPACITY_DIRTY_SCOPE_ENV: OnceLock<(Vec, Arc, TempDir)> = OnceLock::new(); static CAPACITY_DIRTY_SCOPE_INIT: Once = Once::new(); diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 254c3b74c..088802da7 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -28,6 +28,7 @@ pub mod timeout_wrapper; pub mod tonic_service; pub(crate) type StorageDeletedObject = rustfs_ecstore::store_api::DeletedObject; +pub(crate) type StorageGetObjectReader = rustfs_ecstore::store_api::GetObjectReader; pub(crate) type StorageObjectInfo = rustfs_ecstore::store_api::ObjectInfo; pub(crate) type StorageObjectOptions = rustfs_ecstore::store_api::ObjectOptions; pub(crate) type StorageObjectToDelete = rustfs_ecstore::store_api::ObjectToDelete; diff --git a/rustfs/src/table_catalog.rs b/rustfs/src/table_catalog.rs index 12a11290a..8252d62fe 100644 --- a/rustfs/src/table_catalog.rs +++ b/rustfs/src/table_catalog.rs @@ -43,10 +43,7 @@ use rustfs_ecstore::bucket::{ }; use rustfs_ecstore::disk::RUSTFS_META_BUCKET; use rustfs_ecstore::error::{Error as EcstoreError, StorageError}; -use rustfs_ecstore::{ - set_disk::get_lock_acquire_timeout, - store_api::{DeletedObject, GetObjectReader, ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader}, -}; +use rustfs_ecstore::set_disk::get_lock_acquire_timeout; use rustfs_filemeta::FileInfo; use rustfs_storage_api::{ HTTPPreconditions, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo, @@ -59,6 +56,11 @@ use time::{Duration, OffsetDateTime}; use tokio::io::AsyncReadExt; use uuid::Uuid; +use crate::storage::{ + StorageDeletedObject as DeletedObject, StorageGetObjectReader as GetObjectReader, StorageObjectInfo as ObjectInfo, + StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader, +}; + pub(crate) const TABLE_BUCKET_MARKER_CONFIG: &str = BUCKET_TABLE_CONFIG; pub(crate) const RESERVED_CATALOG_OBJECT_MESSAGE: &str = "Object key is reserved for the table catalog"; pub(crate) const TABLE_BUCKET_CATALOG_TYPE: &str = "iceberg-rest";