diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index 7ba2bcc51..a12497500 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -558,13 +558,13 @@ mod tests { async fn format_disk(&self, _endpoint: &Endpoint) -> crate::Result<()> { Ok(()) } - async fn get_bucket_info(&self, _bucket: &str) -> crate::Result> { + async fn get_bucket_info(&self, _bucket: &str) -> crate::Result> { Ok(None) } async fn heal_bucket_metadata(&self, _bucket: &str) -> crate::Result<()> { Ok(()) } - async fn list_buckets(&self) -> crate::Result> { + async fn list_buckets(&self) -> crate::Result> { Ok(vec![]) } async fn object_exists(&self, _bucket: &str, _object: &str) -> crate::Result { diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 2362b1e5b..250a0d92d 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -2393,9 +2393,8 @@ mod tests { use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealTask, HealType}; use rustfs_common::heal_channel::{HealOpts, HealRequestSource}; use rustfs_madmin::heal_commands::HealResultItem; - use rustfs_storage_api::BucketInfo; - use super::super::{DiskStore, Endpoint}; + use super::super::{DiskStore, Endpoint, storage_api::BucketInfo}; struct MockStorage; diff --git a/crates/heal/src/heal/mod.rs b/crates/heal/src/heal/mod.rs index 639db2706..2ed9a1033 100644 --- a/crates/heal/src/heal/mod.rs +++ b/crates/heal/src/heal/mod.rs @@ -19,21 +19,17 @@ pub mod manager; pub mod progress; pub mod resume; pub mod storage; +pub(crate) mod storage_api; pub mod task; pub mod utils; -use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME; -use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint; -use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult}; -use rustfs_ecstore::api::disk::{ - BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, DeleteOptions as EcstoreDeleteOptions, - DiskAPI as EcstoreDiskAPI, DiskStore as EcstoreDiskStore, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, +use storage_api::{ + ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_GLOBAL_LOCAL_DISK_MAP, ECSTORE_RUSTFS_META_BUCKET, + EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore, + EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations, }; #[cfg(test)] -use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk}; -use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError}; -use rustfs_ecstore::api::global::GLOBAL_LOCAL_DISK_MAP as ECSTORE_GLOBAL_LOCAL_DISK_MAP; -use rustfs_ecstore::api::storage::ECStore as EcstoreStore; +use storage_api::{EcstoreDiskOption, ecstore_new_disk}; pub use erasure_healer::ErasureSetHealer; pub use manager::{HealManager, HealOperationsSnapshot, HealPriorityCounts, HealSourceCounts}; @@ -116,6 +112,6 @@ where } } -pub type HealObjectInfo = ::ObjectInfo; -pub type HealObjectOptions = ::ObjectOptions; -pub type HealPutObjReader = ::PutObjectReader; +pub type HealObjectInfo = ::ObjectInfo; +pub type HealObjectOptions = ::ObjectOptions; +pub type HealPutObjReader = ::PutObjectReader; diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index 9b44828ec..6e654ea9b 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -16,13 +16,13 @@ use crate::{Error, Result}; use async_trait::async_trait; use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_madmin::heal_commands::HealResultItem; -use rustfs_storage_api::{ - BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _, - ObjectOperations as _, StorageAdminApi, -}; use std::sync::Arc; use tracing::{debug, error, warn}; +use super::storage_api::{ + BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _, + ObjectOperations as _, StorageAdminApi, +}; use super::{DiskStore, ECStore, Endpoint, StorageError}; pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader}; diff --git a/crates/heal/src/heal/storage_api.rs b/crates/heal/src/heal/storage_api.rs new file mode 100644 index 000000000..d2f39e346 --- /dev/null +++ b/crates/heal/src/heal/storage_api.rs @@ -0,0 +1,29 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME; +pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint; +pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult}; +pub(crate) use rustfs_ecstore::api::disk::{ + BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, DeleteOptions as EcstoreDeleteOptions, + DiskAPI as EcstoreDiskAPI, DiskStore as EcstoreDiskStore, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, +}; +#[cfg(test)] +pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk}; +pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError}; +pub(crate) use rustfs_ecstore::api::global::GLOBAL_LOCAL_DISK_MAP as ECSTORE_GLOBAL_LOCAL_DISK_MAP; +pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore; +pub(crate) use rustfs_storage_api::{ + BucketInfo, BucketOperations, DiskSetSelector, HealOperations, ListOperations, ObjectIO, ObjectOperations, StorageAdminApi, +}; diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index ab21cb55c..844cfd66a 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -2227,10 +2227,10 @@ mod tests { use super::*; use crate::heal::storage::{DiskStatus, HealObjectInfo}; use rustfs_madmin::heal_commands::HealResultItem; - use rustfs_storage_api::BucketInfo; use std::collections::HashMap; use std::sync::Mutex; + use super::super::storage_api::BucketInfo; #[derive(Default)] struct MockStorage { listed: Mutex, diff --git a/crates/heal/tests/endpoint_index_test.rs b/crates/heal/tests/endpoint_index_test.rs index ecec7fc11..6be68e3a9 100644 --- a/crates/heal/tests/endpoint_index_test.rs +++ b/crates/heal/tests/endpoint_index_test.rs @@ -14,13 +14,15 @@ //! test endpoint index settings -use rustfs_ecstore::api::disk::endpoint::Endpoint; -use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints}; -use rustfs_ecstore::api::storage::{ECStore, init_local_disks}; use std::net::SocketAddr; use tempfile::TempDir; use tokio_util::sync::CancellationToken; +#[path = "endpoint_index_test/storage_api.rs"] +mod storage_api; + +use storage_api::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks}; + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn test_endpoint_index_settings() -> anyhow::Result<()> { let temp_dir = TempDir::new()?; diff --git a/crates/heal/tests/endpoint_index_test/storage_api.rs b/crates/heal/tests/endpoint_index_test/storage_api.rs new file mode 100644 index 000000000..3e2a55f68 --- /dev/null +++ b/crates/heal/tests/endpoint_index_test/storage_api.rs @@ -0,0 +1,17 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint; +pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints}; +pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks}; diff --git a/crates/heal/tests/heal_bug_fixes_test.rs b/crates/heal/tests/heal_bug_fixes_test.rs index 25142623d..b18300186 100644 --- a/crates/heal/tests/heal_bug_fixes_test.rs +++ b/crates/heal/tests/heal_bug_fixes_test.rs @@ -12,13 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint}; use rustfs_heal::heal::{ event::{HealEvent, Severity}, task::{HealPriority, HealType}, utils, }; +#[path = "heal_bug_fixes_test/storage_api.rs"] +mod storage_api; + +use storage_api::{BucketInfo, DiskStore, Endpoint}; + #[test] fn test_heal_event_to_heal_request_no_panic() { // Test that invalid pool/set indices don't cause panic @@ -191,13 +195,13 @@ fn test_heal_task_status_atomic_update() { async fn format_disk(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<()> { Ok(()) } - async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result> { + async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result> { Ok(None) } async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> { Ok(()) } - async fn list_buckets(&self) -> rustfs_heal::Result> { + async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(vec![]) } async fn object_exists(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result { @@ -320,7 +324,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { Ok(()) } - async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result> { + async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result> { Ok(None) } @@ -328,7 +332,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { Ok(()) } - async fn list_buckets(&self) -> rustfs_heal::Result> { + async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(Vec::new()) } diff --git a/crates/heal/tests/heal_bug_fixes_test/storage_api.rs b/crates/heal/tests/heal_bug_fixes_test/storage_api.rs new file mode 100644 index 000000000..767572937 --- /dev/null +++ b/crates/heal/tests/heal_bug_fixes_test/storage_api.rs @@ -0,0 +1,16 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint}; +pub(crate) use rustfs_storage_api::BucketInfo; diff --git a/crates/heal/tests/heal_integration_test.rs b/crates/heal/tests/heal_integration_test.rs index 11d9f8153..a1d9cb10d 100644 --- a/crates/heal/tests/heal_integration_test.rs +++ b/crates/heal/tests/heal_integration_test.rs @@ -14,16 +14,11 @@ use http::HeaderMap; use rustfs_common::heal_channel::{HealOpts, HealScanMode}; -use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys; -use rustfs_ecstore::api::disk::endpoint::Endpoint; -use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints}; -use rustfs_ecstore::api::storage::{ECStore, init_local_disks}; use rustfs_heal::heal::{ manager::{HealConfig, HealManager}, storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI}, task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType}, }; -use rustfs_storage_api::{BucketOperations, ObjectIO as _, ObjectOperations as _}; use serial_test::serial; use std::{ path::{Path, PathBuf}, @@ -35,6 +30,14 @@ use tokio_util::sync::CancellationToken; use tracing::info; use walkdir::WalkDir; +#[path = "heal_integration_test/storage_api.rs"] +mod storage_api; + +use storage_api::{ + BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, ObjectIO as _, ObjectOperations as _, + PoolEndpoints, init_bucket_metadata_sys, init_local_disks, +}; + const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25); const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250); const NON_INLINE_TEST_DATA_SIZE: usize = 256 * 1024 + 137; @@ -132,7 +135,7 @@ async fn setup_test_env() -> (Vec, Arc, Arc::GetObjectReader; -pub type ScannerObjectInfo = ::ObjectInfo; -pub type ScannerObjectOptions = ::ObjectOptions; +pub type ScannerObjectInfo = ::ObjectInfo; +pub type ScannerObjectOptions = ::ObjectOptions; pub type ScannerObjectToDelete = ObjectToDelete; pub type ScannerPutObjReader = ::PutObjectReader; diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 88fdd8256..84e13a622 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -38,13 +38,13 @@ use rustfs_config::{ ENV_SCANNER_CYCLE_MAX_OBJECTS, }; use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS}; -use rustfs_storage_api::{BucketOperations, BucketOptions, NamespaceLocking as _}; use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; use tokio::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, instrument, warn}; +use crate::storage_api::{BucketOperations, BucketOptions, NamespaceLocking as _}; use crate::{ ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _, get_lifecycle_config, get_replication_config, read_config, replace_bucket_usage_memory_from_info, save_config, @@ -1148,9 +1148,9 @@ mod tests { } #[async_trait::async_trait] - impl rustfs_storage_api::ObjectIO for MemoryConfigStore { + impl crate::storage_api::ObjectIO for MemoryConfigStore { type Error = EcstoreError; - type RangeSpec = rustfs_storage_api::HTTPRangeSpec; + type RangeSpec = crate::storage_api::HTTPRangeSpec; type HeaderMap = http::HeaderMap; type ObjectOptions = ObjectOptions; type ObjectInfo = ObjectInfo; @@ -1161,7 +1161,7 @@ mod tests { &self, bucket: &str, object: &str, - _range: Option, + _range: Option, _h: http::HeaderMap, _opts: &ObjectOptions, ) -> EcstoreResult { diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 85065ddb8..ca3de5d13 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -27,7 +27,6 @@ use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, e #[cfg(test)] use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS}; use rustfs_filemeta::FileMeta; -use rustfs_storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi}; use rustfs_utils::path::path_join_buf; use s3s::dto::{BucketLifecycleConfiguration, ReplicationConfiguration}; use std::collections::HashMap; @@ -43,6 +42,7 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, error, warn}; use crate::ScannerObjectInfo as ObjectInfo; +use crate::storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi}; use crate::{ BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result, ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _, diff --git a/crates/scanner/src/storage_api.rs b/crates/scanner/src/storage_api.rs new file mode 100644 index 000000000..8eb0338dd --- /dev/null +++ b/crates/scanner/src/storage_api.rs @@ -0,0 +1,78 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys; +pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc; +pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{ + apply_expiry_rule as ecstore_apply_expiry_rule, apply_transition_rule as ecstore_apply_transition_rule, + get_global_expiry_state as ecstore_get_global_expiry_state, +}; +pub(crate) use rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator as EcstoreEvaluator; +pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{ + Event as EcstoreEvent, Lifecycle as EcstoreLifecycle, ObjectOpts as EcstoreObjectOpts, + TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE, +}; +pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{ + get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config, + get_replication_config as ecstore_get_replication_config, +}; +pub(crate) use rustfs_ecstore::api::bucket::replication::{ + ReplicationConfig as EcstoreReplicationConfig, ReplicationConfigurationExt as EcstoreReplicationConfigurationExt, + ReplicationHealQueueResult as EcstoreReplicationHealQueueResult, + ReplicationQueueAdmission as EcstoreReplicationQueueAdmission, + queue_replication_heal_internal as ecstore_queue_replication_heal_internal, +}; +pub(crate) use rustfs_ecstore::api::bucket::versioning::VersioningApi as EcstoreVersioningApi; +pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys as EcstoreBucketVersioningSys; +pub(crate) use rustfs_ecstore::api::cache::{ + ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw, +}; +pub(crate) use rustfs_ecstore::api::capacity::{ + is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object, + path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path, +}; +pub(crate) use rustfs_ecstore::api::config::com::{read_config as ecstore_read_config, save_config as ecstore_save_config}; +#[cfg(test)] +pub(crate) use rustfs_ecstore::api::config::init as ecstore_config_init; +pub(crate) use rustfs_ecstore::api::config::storageclass::{ + RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD, +}; +pub(crate) use rustfs_ecstore::api::data_usage::replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info; +#[cfg(test)] +pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint; +pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult}; +pub(crate) use rustfs_ecstore::api::disk::{ + BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, Disk as EcstoreDisk, DiskAPI as EcstoreDiskAPI, + DiskInfo as EcstoreDiskInfo, DiskInfoOptions as EcstoreDiskInfoOptions, DiskLocation as EcstoreDiskLocation, + RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE, + ScanGuard as EcstoreScanGuard, +}; +#[cfg(test)] +pub(crate) use rustfs_ecstore::api::disk::{ + DiskOption as EcstoreDiskOption, DiskStore as EcstoreDiskStore, new_disk as ecstore_new_disk, +}; +pub(crate) use rustfs_ecstore::api::error::{ + Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError, +}; +pub(crate) use rustfs_ecstore::api::global::{ + get_global_tier_config_mgr as ecstore_get_global_tier_config_mgr, is_erasure as ecstore_is_erasure, + is_erasure_sd as ecstore_is_erasure_sd, resolve_object_store_handle as ecstore_resolve_object_store_handle, +}; +pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks; +pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore; +pub(crate) use rustfs_ecstore::api::tier::tier_config::TierConfig as EcstoreTierConfig; +pub(crate) use rustfs_storage_api::{ + BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, HTTPRangeSpec, NamespaceLocking, ObjectIO, ObjectOperations, + ObjectToDelete, StorageAdminApi, +}; diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index c1e4c0a1e..3b78c9ef3 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -14,25 +14,6 @@ use futures::FutureExt; use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT; -use rustfs_ecstore::api::bucket::lifecycle::{ - bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, init_background_expiry}, - lifecycle::TransitionOptions, -}; -use rustfs_ecstore::api::bucket::metadata::BUCKET_LIFECYCLE_CONFIG; -use rustfs_ecstore::api::bucket::metadata_sys::{ - get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata, -}; -use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys; -use rustfs_ecstore::api::capacity::path2_bucket_object_with_base_path; -use rustfs_ecstore::api::client::transition_api::{ReadCloser, ReaderImpl}; -use rustfs_ecstore::api::disk::{DiskAPI as _, DiskOption, STORAGE_FORMAT_FILE, endpoint::Endpoint, new_disk}; -use rustfs_ecstore::api::global::get_global_tier_config_mgr; -use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints}; -use rustfs_ecstore::api::storage::{ECStore, init_local_disks}; -use rustfs_ecstore::api::tier::tier_config::{TierConfig, TierMinIO, TierType}; -use rustfs_ecstore::api::tier::warm_backend::{ - WarmBackend as ScannerWarmBackend, WarmBackendGetOpts, build_transition_put_options, -}; use rustfs_filemeta::FileMeta; use rustfs_scanner::scanner_folder::ScannerItem; use rustfs_scanner::scanner_io::ScannerIODisk; @@ -40,9 +21,6 @@ use rustfs_scanner::{ ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader, scanner::init_data_scanner, }; -use rustfs_storage_api::{ - BucketOperations, ListOperations as _, MakeBucketOptions, MultipartOperations as _, ObjectIO as _, ObjectOperations as _, -}; use rustfs_utils::path::path_join_buf; use s3s::dto::RestoreRequest; use serial_test::serial; @@ -61,6 +39,17 @@ use tokio_util::sync::CancellationToken; use tracing::info; use uuid::Uuid; +mod storage_api; + +use storage_api::{ + BUCKET_LIFECYCLE_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, CompletePart, DiskAPI as _, DiskOption, + ECStore, Endpoint, EndpointServerPools, Endpoints, ListOperations as _, MakeBucketOptions, MultipartOperations as _, + ObjectIO as _, ObjectOperations as _, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, ScannerWarmBackend, + TierConfig, TierMinIO, TierType, TransitionOptions, WarmBackendGetOpts, build_transition_put_options, + enqueue_transition_for_existing_objects, get_bucket_metadata, get_global_tier_config_mgr, init_background_expiry, + init_bucket_metadata_sys, init_local_disks, new_disk, path2_bucket_object_with_base_path, update_bucket_metadata, +}; + static GLOBAL_ENV: OnceLock<(Vec, Arc)> = OnceLock::new(); static INIT: Once = Once::new(); const TRANSITION_WAIT_TIMEOUT: Duration = Duration::from_secs(15); @@ -134,7 +123,7 @@ async fn setup_test_env() -> (Vec, Arc) { // init bucket metadata system let buckets_list = ecstore - .list_bucket(&rustfs_storage_api::BucketOptions { + .list_bucket(&BucketOptions { no_metadata: true, ..Default::default() }) @@ -200,7 +189,7 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec, Arc"$DIRECT_ECSTORE_IMPORT_HITS_FILE" @@ -1106,7 +1110,7 @@ fi --glob '!**/ecstore_compat.rs' \ --glob '!**/ecstore_test_compat.rs' \ --glob '!**/ecstore_test_compat/**' | - rg -v '^(fuzz/fuzz_targets/bucket_validation\.rs|fuzz/fuzz_targets/path_containment\.rs|crates/e2e_test/src/reliant/grpc_lock_client\.rs|crates/e2e_test/src/reliant/node_interact_test\.rs|crates/e2e_test/src/replication_extension_test\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/endpoint_index_test\.rs|crates/heal/tests/heal_bug_fixes_test\.rs|crates/heal/tests/heal_integration_test\.rs|crates/iam/src/(lib|runtime_sources)\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs|rustfs/src/admin/mod\.rs|rustfs/src/app/mod\.rs|rustfs/src/storage/mod\.rs):' || true + rg -v '^(fuzz/fuzz_targets/bucket_validation\.rs|fuzz/fuzz_targets/path_containment\.rs|crates/e2e_test/src/reliant/grpc_lock_client\.rs|crates/e2e_test/src/reliant/node_interact_test\.rs|crates/e2e_test/src/replication_extension_test\.rs|crates/heal/src/heal/storage_api\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/iam/src/(lib|runtime_sources)\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs|rustfs/src/admin/mod\.rs|rustfs/src/app/mod\.rs|rustfs/src/storage/mod\.rs):' || true ) >"$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" if [[ -s "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" ]]; then @@ -1116,17 +1120,17 @@ fi ( cd "$ROOT_DIR" rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \ - crates/heal/src/heal/mod.rs \ - crates/heal/tests/endpoint_index_test.rs \ - crates/heal/tests/heal_bug_fixes_test.rs \ - crates/heal/tests/heal_integration_test.rs \ + crates/heal/src/heal/storage_api.rs \ + crates/heal/tests/endpoint_index_test/storage_api.rs \ + crates/heal/tests/heal_bug_fixes_test/storage_api.rs \ + crates/heal/tests/heal_integration_test/storage_api.rs \ crates/iam/src/lib.rs \ crates/notify/src/lib.rs \ crates/obs/src/metrics/mod.rs \ crates/protocols/src/swift/mod.rs \ crates/s3select-api/src/lib.rs \ - crates/scanner/src/lib.rs \ - crates/scanner/tests/lifecycle_integration_test.rs \ + crates/scanner/src/storage_api.rs \ + crates/scanner/tests/storage_api/mod.rs \ crates/e2e_test/src/reliant/grpc_lock_client.rs \ crates/e2e_test/src/reliant/node_interact_test.rs \ crates/e2e_test/src/replication_extension_test.rs \ @@ -1144,17 +1148,17 @@ fi ( cd "$ROOT_DIR" rg -n --with-filename '^(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+\s*;|^(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+::\*\s*;' \ - crates/heal/src/heal/mod.rs \ - crates/heal/tests/endpoint_index_test.rs \ - crates/heal/tests/heal_bug_fixes_test.rs \ - crates/heal/tests/heal_integration_test.rs \ + crates/heal/src/heal/storage_api.rs \ + crates/heal/tests/endpoint_index_test/storage_api.rs \ + crates/heal/tests/heal_bug_fixes_test/storage_api.rs \ + crates/heal/tests/heal_integration_test/storage_api.rs \ crates/iam/src/lib.rs \ crates/notify/src/lib.rs \ crates/obs/src/metrics/mod.rs \ crates/protocols/src/swift/mod.rs \ crates/s3select-api/src/lib.rs \ - crates/scanner/src/lib.rs \ - crates/scanner/tests/lifecycle_integration_test.rs \ + crates/scanner/src/storage_api.rs \ + crates/scanner/tests/storage_api/mod.rs \ crates/e2e_test/src/reliant/grpc_lock_client.rs \ crates/e2e_test/src/reliant/node_interact_test.rs \ crates/e2e_test/src/replication_extension_test.rs \ @@ -1172,17 +1176,17 @@ fi ( cd "$ROOT_DIR" rg -n --with-filename 'rustfs_ecstore::api::[a-z_]+::' \ - crates/heal/src/heal/mod.rs \ - crates/heal/tests/endpoint_index_test.rs \ - crates/heal/tests/heal_bug_fixes_test.rs \ - crates/heal/tests/heal_integration_test.rs \ + crates/heal/src/heal/storage_api.rs \ + crates/heal/tests/endpoint_index_test/storage_api.rs \ + crates/heal/tests/heal_bug_fixes_test/storage_api.rs \ + crates/heal/tests/heal_integration_test/storage_api.rs \ crates/iam/src/lib.rs \ crates/notify/src/lib.rs \ crates/obs/src/metrics/mod.rs \ crates/protocols/src/swift/mod.rs \ crates/s3select-api/src/lib.rs \ - crates/scanner/src/lib.rs \ - crates/scanner/tests/lifecycle_integration_test.rs \ + crates/scanner/src/storage_api.rs \ + crates/scanner/tests/storage_api/mod.rs \ crates/e2e_test/src/reliant/grpc_lock_client.rs \ crates/e2e_test/src/reliant/node_interact_test.rs \ crates/e2e_test/src/replication_extension_test.rs \ @@ -1306,7 +1310,7 @@ fi crates/scanner/src \ --glob '*.rs' \ --glob '!**/ecstore_compat.rs' | - rg -v '^(crates/heal/src/heal/mod.rs|crates/iam/src/(lib|runtime_sources).rs|crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs|crates/scanner/src/lib.rs):' || true + rg -v '^(crates/heal/src/heal/storage_api.rs|crates/iam/src/(lib|runtime_sources).rs|crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs|crates/scanner/src/storage_api.rs):' || true ) >"$EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE" if [[ -s "$EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE" ]]; then @@ -1594,7 +1598,7 @@ fi crates/scanner/tests \ crates/e2e_test/src \ --glob '*.rs' \ - | rg -v '^(crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)\.rs|crates/scanner/tests/lifecycle_integration_test\.rs):' || true + | rg -v '^(crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs|crates/scanner/tests/storage_api/mod\.rs):' || true ) >"$EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE" if [[ -s "$EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE" ]]; then @@ -1618,7 +1622,7 @@ fi rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \ crates/heal/src crates/iam/src crates/notify/src crates/obs/src crates/protocols/src crates/s3select-api/src crates/scanner/src \ --glob '*.rs' | - rg -v '^(crates/heal/src/heal/mod.rs|crates/iam/src/lib.rs|crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs|crates/scanner/src/lib.rs):' || true + rg -v '^(crates/heal/src/heal/storage_api.rs|crates/iam/src/lib.rs|crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs|crates/scanner/src/storage_api.rs):' || true ) >"$EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE" if [[ -s "$EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE" ]]; then @@ -2020,6 +2024,54 @@ if [[ -s "$SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE" ]]; then report_failure "scanner storage compatibility must expose bucket contracts as explicit aliases: $(paste -sd '; ' "$SCANNER_BUCKET_STORAGE_COMPAT_MODULE_HITS_FILE")" fi +( + cd "$ROOT_DIR" + rg -n --with-filename 'rustfs_ecstore::api::|rustfs_storage_api' \ + crates/scanner/src \ + --glob '*.rs' | + rg -v '^crates/scanner/src/storage_api\.rs:' || true +) >"$SCANNER_STORAGE_API_SOURCE_BYPASS_HITS_FILE" + +if [[ -s "$SCANNER_STORAGE_API_SOURCE_BYPASS_HITS_FILE" ]]; then + report_failure "scanner source files must route ECStore and storage-api symbols through scanner::storage_api: $(paste -sd '; ' "$SCANNER_STORAGE_API_SOURCE_BYPASS_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n --with-filename 'rustfs_ecstore::api::|rustfs_storage_api' \ + crates/scanner/tests \ + --glob '*.rs' | + rg -v '^crates/scanner/tests/storage_api/mod\.rs:' || true +) >"$SCANNER_STORAGE_API_TEST_BYPASS_HITS_FILE" + +if [[ -s "$SCANNER_STORAGE_API_TEST_BYPASS_HITS_FILE" ]]; then + report_failure "scanner tests must route ECStore and storage-api symbols through scanner test storage_api: $(paste -sd '; ' "$SCANNER_STORAGE_API_TEST_BYPASS_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n --with-filename 'rustfs_ecstore::api::|rustfs_storage_api' \ + crates/heal/src/heal \ + --glob '*.rs' | + rg -v '^crates/heal/src/heal/storage_api\.rs:' || true +) >"$HEAL_STORAGE_API_SOURCE_BYPASS_HITS_FILE" + +if [[ -s "$HEAL_STORAGE_API_SOURCE_BYPASS_HITS_FILE" ]]; then + report_failure "heal source files must route ECStore and storage-api symbols through heal::storage_api: $(paste -sd '; ' "$HEAL_STORAGE_API_SOURCE_BYPASS_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n --with-filename 'rustfs_ecstore::api::|rustfs_storage_api' \ + crates/heal/tests \ + --glob '*.rs' | + rg -v '^crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)/storage_api\.rs:' || true +) >"$HEAL_STORAGE_API_TEST_BYPASS_HITS_FILE" + +if [[ -s "$HEAL_STORAGE_API_TEST_BYPASS_HITS_FILE" ]]; then + report_failure "heal tests must route ECStore and storage-api symbols through heal test storage_api: $(paste -sd '; ' "$HEAL_STORAGE_API_TEST_BYPASS_HITS_FILE")" +fi + ( cd "$ROOT_DIR" if [[ -f crates/notify/src/storage_compat.rs ]]; then