diff --git a/crates/e2e_test/src/reliant/node_interact_test.rs b/crates/e2e_test/src/reliant/node_interact_test.rs index a96a78916..f9165fede 100644 --- a/crates/e2e_test/src/reliant/node_interact_test.rs +++ b/crates/e2e_test/src/reliant/node_interact_test.rs @@ -14,8 +14,8 @@ // limitations under the License. use crate::common::workspace_root; -use crate::storage_compat::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; use crate::storage_compat::{VolumeInfo, WalkDirOptions}; +use crate::storage_compat::{gen_tonic_signature_interceptor, node_service_time_out_client}; use futures::future::join_all; use rmp_serde::{Deserializer, Serializer}; use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter}; @@ -53,9 +53,7 @@ async fn ping() -> Result<(), Box> { assert!(decoded_payload.is_ok()); // Create client - let mut client = - node_service_time_out_client(&CLUSTER_ADDR.to_string(), TonicInterceptor::Signature(gen_tonic_signature_interceptor())) - .await?; + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), gen_tonic_signature_interceptor()).await?; // Construct PingRequest let request = Request::new(PingRequest { @@ -80,9 +78,7 @@ async fn ping() -> Result<(), Box> { #[tokio::test] #[ignore = "requires running RustFS server at localhost:9000"] async fn make_volume() -> Result<(), Box> { - let mut client = - node_service_time_out_client(&CLUSTER_ADDR.to_string(), TonicInterceptor::Signature(gen_tonic_signature_interceptor())) - .await?; + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), gen_tonic_signature_interceptor()).await?; let request = Request::new(MakeVolumeRequest { disk: "data".to_string(), volume: "dandan".to_string(), @@ -100,9 +96,7 @@ async fn make_volume() -> Result<(), Box> { #[tokio::test] #[ignore = "requires running RustFS server at localhost:9000"] async fn list_volumes() -> Result<(), Box> { - let mut client = - node_service_time_out_client(&CLUSTER_ADDR.to_string(), TonicInterceptor::Signature(gen_tonic_signature_interceptor())) - .await?; + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), gen_tonic_signature_interceptor()).await?; let request = Request::new(ListVolumesRequest { disk: "data".to_string(), }); @@ -132,9 +126,7 @@ async fn walk_dir() -> Result<(), Box> { let (rd, mut wr) = tokio::io::duplex(1024); let mut buf = Vec::new(); opts.serialize(&mut Serializer::new(&mut buf))?; - let mut client = - node_service_time_out_client(&CLUSTER_ADDR.to_string(), TonicInterceptor::Signature(gen_tonic_signature_interceptor())) - .await?; + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), gen_tonic_signature_interceptor()).await?; let disk_path = std::env::var_os("RUSTFS_DISK_PATH").map(PathBuf::from).unwrap_or_else(|| { let mut path = workspace_root(); path.push("target"); @@ -187,9 +179,7 @@ async fn walk_dir() -> Result<(), Box> { #[tokio::test] #[ignore = "requires running RustFS server at localhost:9000"] async fn read_all() -> Result<(), Box> { - let mut client = - node_service_time_out_client(&CLUSTER_ADDR.to_string(), TonicInterceptor::Signature(gen_tonic_signature_interceptor())) - .await?; + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), gen_tonic_signature_interceptor()).await?; let request = Request::new(ReadAllRequest { disk: "data".to_string(), volume: "ff".to_string(), @@ -207,9 +197,7 @@ async fn read_all() -> Result<(), Box> { #[tokio::test] #[ignore = "requires running RustFS server at localhost:9000"] async fn storage_info() -> Result<(), Box> { - let mut client = - node_service_time_out_client(&CLUSTER_ADDR.to_string(), TonicInterceptor::Signature(gen_tonic_signature_interceptor())) - .await?; + let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string(), gen_tonic_signature_interceptor()).await?; let request = Request::new(LocalStorageInfoRequest { metrics: true }); let response = client.local_storage_info(request).await?.into_inner(); diff --git a/crates/e2e_test/src/storage_compat.rs b/crates/e2e_test/src/storage_compat.rs index 68b701de4..693d71b3a 100644 --- a/crates/e2e_test/src/storage_compat.rs +++ b/crates/e2e_test/src/storage_compat.rs @@ -19,7 +19,9 @@ pub(crate) type TonicInterceptor = rustfs_ecstore::api::rpc::TonicInterceptor; pub(crate) type VolumeInfo = rustfs_ecstore::api::disk::VolumeInfo; pub(crate) type WalkDirOptions = rustfs_ecstore::api::disk::WalkDirOptions; -pub(crate) use rustfs_ecstore::api::rpc::gen_tonic_signature_interceptor; +pub(crate) fn gen_tonic_signature_interceptor() -> TonicInterceptor { + TonicInterceptor::Signature(rustfs_ecstore::api::rpc::gen_tonic_signature_interceptor()) +} pub(crate) async fn node_service_time_out_client( addr: &String, diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 7748c470f..d6ace40ed 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -42,7 +42,7 @@ pub mod capacity { } pub mod client { - pub use crate::client::{admin_handler_utils, object_api_utils, transition_api}; + pub use crate::client::{admin_handler_utils, api_put_object, object_api_utils, transition_api}; } pub mod cluster { diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index d8fed1446..50cac731c 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -20,6 +20,7 @@ use crate::heal::{ use crate::{Error, Result}; use metrics::{counter, gauge}; use rustfs_common::heal_channel::{HealAdmissionDropReason, HealAdmissionResult, HealRequestSource}; +use rustfs_ecstore::api::disk::DiskAPI as _; use rustfs_madmin::heal_commands::HealResultItem; use std::{ collections::{BinaryHeap, HashMap}, @@ -33,7 +34,7 @@ use tokio::{ use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; -use super::storage_compat::{DiskAPI, DiskError, GLOBAL_LOCAL_DISK_MAP}; +use super::storage_compat::{DiskError, GLOBAL_LOCAL_DISK_MAP}; const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60); const LOG_COMPONENT_HEAL: &str = "heal"; diff --git a/crates/heal/src/heal/resume.rs b/crates/heal/src/heal/resume.rs index 48ac37d64..9dac25a58 100644 --- a/crates/heal/src/heal/resume.rs +++ b/crates/heal/src/heal/resume.rs @@ -13,6 +13,7 @@ // limitations under the License. use crate::{Error, Result}; +use rustfs_ecstore::api::disk::DiskAPI as _; use serde::{Deserialize, Serialize}; use std::path::Path; use std::sync::Arc; @@ -21,7 +22,7 @@ use tokio::sync::RwLock; use tracing::{debug, warn}; use uuid::Uuid; -use super::storage_compat::{BUCKET_META_PREFIX, DiskAPI, DiskError, DiskStore, RUSTFS_META_BUCKET}; +use super::storage_compat::{BUCKET_META_PREFIX, DiskError, DiskStore, RUSTFS_META_BUCKET}; const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_RESUME: &str = "resume"; diff --git a/crates/heal/src/heal/storage_compat.rs b/crates/heal/src/heal/storage_compat.rs index 75019eda0..788ce1e40 100644 --- a/crates/heal/src/heal/storage_compat.rs +++ b/crates/heal/src/heal/storage_compat.rs @@ -22,9 +22,17 @@ pub(crate) type ECStore = rustfs_ecstore::api::storage::ECStore; pub(crate) type EcstoreError = rustfs_ecstore::api::error::Error; pub(crate) type Endpoint = rustfs_ecstore::api::disk::endpoint::Endpoint; pub(crate) type StorageError = rustfs_ecstore::api::error::StorageError; +pub(crate) type LocalDiskMap = std::collections::HashMap>; -pub(crate) use rustfs_ecstore::api::disk::DiskAPI; -pub(crate) use rustfs_ecstore::api::global::GLOBAL_LOCAL_DISK_MAP; +pub(crate) struct GlobalLocalDiskMap; + +pub(crate) static GLOBAL_LOCAL_DISK_MAP: GlobalLocalDiskMap = GlobalLocalDiskMap; + +impl GlobalLocalDiskMap { + pub(crate) async fn read(&self) -> tokio::sync::RwLockReadGuard<'static, LocalDiskMap> { + rustfs_ecstore::api::global::GLOBAL_LOCAL_DISK_MAP.read().await + } +} #[cfg(test)] pub(crate) type DiskOption = rustfs_ecstore::api::disk::DiskOption; diff --git a/crates/heal/tests/common/storage_compat.rs b/crates/heal/tests/common/storage_compat.rs index 35d22bc95..e3ad79fcf 100644 --- a/crates/heal/tests/common/storage_compat.rs +++ b/crates/heal/tests/common/storage_compat.rs @@ -19,10 +19,14 @@ use std::sync::Arc; pub(crate) type DiskStore = rustfs_ecstore::api::disk::DiskStore; pub(crate) type ECStore = rustfs_ecstore::api::storage::ECStore; pub(crate) type Endpoint = rustfs_ecstore::api::disk::endpoint::Endpoint; +pub(crate) type EndpointServerPools = rustfs_ecstore::api::layout::EndpointServerPools; pub(crate) type Endpoints = rustfs_ecstore::api::layout::Endpoints; pub(crate) type PoolEndpoints = rustfs_ecstore::api::layout::PoolEndpoints; -pub(crate) use rustfs_ecstore::api::layout::EndpointServerPools; +#[allow(non_snake_case)] +pub(crate) fn EndpointServerPools(pools: Vec) -> EndpointServerPools { + rustfs_ecstore::api::layout::EndpointServerPools::from(pools) +} pub(crate) async fn init_bucket_metadata_sys(api: Arc, buckets: Vec) { rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await; diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 23f302b55..b947c99b9 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -39,6 +39,8 @@ use rustfs_config::{ ENV_SCANNER_CYCLE_MAX_OBJECTS, }; use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS}; +use rustfs_ecstore::api::bucket::lifecycle::lifecycle::Lifecycle as _; +use rustfs_ecstore::api::bucket::replication::ReplicationConfigurationExt as _; use rustfs_storage_api::{BucketOperations, BucketOptions, NamespaceLocking as _}; use serde::{Deserialize, Serialize}; use tokio::sync::mpsc; @@ -47,8 +49,8 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, instrument, warn}; use crate::storage_compat::{ - ECStore, EcstoreError, Lifecycle as _, RUSTFS_META_BUCKET, ReplicationConfigurationExt as _, get_lifecycle_config, - get_replication_config, is_erasure_sd, read_config, replace_bucket_usage_memory_from_info, save_config, + ECStore, EcstoreError, RUSTFS_META_BUCKET, get_lifecycle_config, get_replication_config, is_erasure_sd, read_config, + replace_bucket_usage_memory_from_info, save_config, }; const LOG_COMPONENT_SCANNER: &str = "scanner"; diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index f04b4d50c..e9320aea7 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -39,6 +39,10 @@ use rustfs_common::metrics::{ IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource, UpdateCurrentPathFn, current_path_updater, global_metrics, }; +use rustfs_ecstore::api::bucket::lifecycle::lifecycle::Lifecycle as _; +use rustfs_ecstore::api::bucket::replication::ReplicationConfigurationExt as _; +use rustfs_ecstore::api::bucket::versioning::VersioningApi as _; +use rustfs_ecstore::api::disk::DiskAPI as _; use rustfs_filemeta::{ MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ReplicateObjectInfo, ReplicationStatusType, ReplicationType, }; @@ -51,10 +55,10 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, error, warn}; use crate::storage_compat::{ - BucketVersioningSys, Disk, DiskAPI as _, DiskError, DiskInfoOptions, Evaluator, Event, GLOBAL_ExpiryState, LcEventSrc, - Lifecycle, ListPathRawOptions, ObjectOpts, ReplicationConfig, ReplicationConfigurationExt as _, ReplicationQueueAdmission, - StorageError, VersioningApi, apply_expiry_rule, apply_transition_rule, is_erasure, is_reserved_or_invalid_bucket, - list_path_raw, path2_bucket_object, path2_bucket_object_with_base_path, queue_replication_heal_internal, + BucketVersioningSys, Disk, DiskError, DiskInfoOptions, Evaluator, Event, LcEventSrc, ListPathRawOptions, ObjectOpts, + ReplicationConfig, ReplicationQueueAdmission, StorageError, apply_expiry_rule, apply_transition_rule, + enqueue_global_newer_noncurrent, is_erasure, is_reserved_or_invalid_bucket, list_path_raw, path2_bucket_object, + path2_bucket_object_with_base_path, queue_replication_heal_internal, }; use crate::{ScannerObjectInfo as ObjectInfo, ScannerObjectToDelete as ObjectToDelete}; @@ -896,11 +900,7 @@ impl ScannerItem { let action = event.action; let count = u64::try_from(to_delete_objs.len()).unwrap_or(u64::MAX); let done_ilm = Metrics::time_ilm(action); - let queued = GLOBAL_ExpiryState - .write() - .await - .enqueue_by_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner) - .await; + let queued = enqueue_global_newer_noncurrent(&self.bucket, to_delete_objs, event, &LcEventSrc::Scanner).await; if record_scanner_ilm_action_if_queued(global_metrics(), action, count, queued) { done_ilm(count)(); remaining_versions = remaining_versions.saturating_sub(noncurrent_accounting.len()); diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index fe8b7eaa7..44ef99915 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -26,6 +26,10 @@ use rustfs_common::heal_channel::HealScanMode; use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics}; #[cfg(test)] use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS}; +use rustfs_ecstore::api::bucket::lifecycle::lifecycle::Lifecycle as _; +use rustfs_ecstore::api::bucket::replication::ReplicationConfigurationExt as _; +use rustfs_ecstore::api::bucket::versioning::VersioningApi as _; +use rustfs_ecstore::api::disk::DiskAPI as _; use rustfs_filemeta::FileMeta; use rustfs_storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi}; use rustfs_utils::path::path_join_buf; @@ -44,10 +48,9 @@ use tracing::{debug, error, warn}; use crate::ScannerObjectInfo as ObjectInfo; use crate::storage_compat::{ - BucketTargetSys, BucketVersioningSys, Disk, DiskAPI, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result, - GLOBAL_ExpiryState, GLOBAL_TierConfigMgr, Lifecycle, ReplicationConfig, ReplicationConfigurationExt, STORAGE_FORMAT_FILE, - SetDisks, StorageError, VersioningApi as _, get_lifecycle_config, get_object_lock_config, get_replication_config, - resolve_scanner_object_store_handle, storageclass, + BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result, + ReplicationConfig, STORAGE_FORMAT_FILE, SetDisks, StorageError, enqueue_global_free_version, get_lifecycle_config, + get_object_lock_config, get_replication_config, list_global_tiers, resolve_scanner_object_store_handle, storageclass, }; pub(crate) const SCANNER_SKIP_FILE_ERROR: &str = "skip file"; @@ -1349,10 +1352,7 @@ impl ScannerIODisk for Disk { let mut size_summary = SizeSummary::default(); - let tiers = { - let tier_config_mgr = GLOBAL_TierConfigMgr.read().await; - tier_config_mgr.list_tiers() - }; + let tiers = list_global_tiers().await; for tier in tiers.iter() { size_summary.tier_stats.insert(tier.name.clone(), TierStats::default()); @@ -1374,9 +1374,8 @@ impl ScannerIODisk for Disk { item.apply_actions(object_infos, lock_config, &mut size_summary).await; if !free_version_infos.is_empty() { - let mut expiry_state = GLOBAL_ExpiryState.write().await; for oi in free_version_infos { - expiry_state.enqueue_free_version(oi).await; + enqueue_global_free_version(oi).await; } } diff --git a/crates/scanner/src/storage_compat.rs b/crates/scanner/src/storage_compat.rs index f226472e3..8941d772b 100644 --- a/crates/scanner/src/storage_compat.rs +++ b/crates/scanner/src/storage_compat.rs @@ -23,32 +23,26 @@ pub(crate) const STORAGE_FORMAT_FILE: &str = rustfs_ecstore::api::disk::STORAGE_ pub(crate) const TRANSITION_COMPLETE: &str = rustfs_ecstore::api::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; pub(crate) type Disk = rustfs_ecstore::api::disk::Disk; +#[cfg(test)] +pub(crate) type DiskStore = rustfs_ecstore::api::disk::DiskStore; pub(crate) type DiskError = rustfs_ecstore::api::disk::error::DiskError; pub(crate) type ECStore = rustfs_ecstore::api::storage::ECStore; pub(crate) type EcstoreError = rustfs_ecstore::api::error::Error; pub(crate) type EcstoreResult = rustfs_ecstore::api::error::Result; pub(crate) type ListPathRawOptions = rustfs_ecstore::api::cache::ListPathRawOptions; +pub(crate) type BucketTargetSys = rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys; +pub(crate) type BucketVersioningSys = rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys; +pub(crate) type DiskInfoOptions = rustfs_ecstore::api::disk::DiskInfoOptions; +pub(crate) type Evaluator = rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator; +pub(crate) type Event = rustfs_ecstore::api::bucket::lifecycle::lifecycle::Event; +pub(crate) type LcEventSrc = rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; +pub(crate) type ObjectOpts = rustfs_ecstore::api::bucket::lifecycle::lifecycle::ObjectOpts; +pub(crate) type ReplicationConfig = rustfs_ecstore::api::bucket::replication::ReplicationConfig; +pub(crate) type ReplicationHealQueueResult = rustfs_ecstore::api::bucket::replication::ReplicationHealQueueResult; +pub(crate) type ReplicationQueueAdmission = rustfs_ecstore::api::bucket::replication::ReplicationQueueAdmission; pub(crate) type SetDisks = rustfs_ecstore::api::set_disk::SetDisks; pub(crate) type StorageError = rustfs_ecstore::api::error::StorageError; -pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys; -pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; -pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{ - GLOBAL_ExpiryState, apply_expiry_rule, apply_transition_rule, -}; -pub(crate) use rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator; -pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{Event, Lifecycle, ObjectOpts}; -pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{ - get_lifecycle_config, get_object_lock_config, get_replication_config, -}; -pub(crate) use rustfs_ecstore::api::bucket::replication::{ - ReplicationConfig, ReplicationConfigurationExt, ReplicationQueueAdmission, queue_replication_heal_internal, -}; -pub(crate) use rustfs_ecstore::api::bucket::versioning::VersioningApi; -pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys; -pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskInfoOptions}; -pub(crate) use rustfs_ecstore::api::global::GLOBAL_TierConfigMgr; - pub type ScannerGetObjectReader = ::GetObjectReader; pub type ScannerObjectInfo = ::ObjectInfo; pub type ScannerObjectOptions = ::ObjectOptions; @@ -61,10 +55,79 @@ pub(crate) mod storageclass { } #[cfg(test)] -pub(crate) use rustfs_ecstore::api::config::init as init_ecstore_config_for_scanner_tests; +pub(crate) fn init_ecstore_config_for_scanner_tests() { + rustfs_ecstore::api::config::init(); +} #[cfg(test)] -pub(crate) use rustfs_ecstore::api::disk::{DiskOption, endpoint::Endpoint, new_disk}; +pub(crate) type DiskOption = rustfs_ecstore::api::disk::DiskOption; +#[cfg(test)] +pub(crate) type Endpoint = rustfs_ecstore::api::disk::endpoint::Endpoint; + +#[cfg(test)] +pub(crate) async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> rustfs_ecstore::api::disk::error::Result { + rustfs_ecstore::api::disk::new_disk(ep, opt).await +} + +pub(crate) async fn get_lifecycle_config( + bucket: &str, +) -> EcstoreResult<(s3s::dto::BucketLifecycleConfiguration, time::OffsetDateTime)> { + rustfs_ecstore::api::bucket::metadata_sys::get_lifecycle_config(bucket).await +} + +pub(crate) async fn get_object_lock_config( + bucket: &str, +) -> EcstoreResult<(s3s::dto::ObjectLockConfiguration, time::OffsetDateTime)> { + rustfs_ecstore::api::bucket::metadata_sys::get_object_lock_config(bucket).await +} + +pub(crate) async fn get_replication_config( + bucket: &str, +) -> EcstoreResult<(s3s::dto::ReplicationConfiguration, time::OffsetDateTime)> { + rustfs_ecstore::api::bucket::metadata_sys::get_replication_config(bucket).await +} + +pub(crate) async fn apply_transition_rule(event: &Event, src: &LcEventSrc, oi: &ScannerObjectInfo) -> bool { + rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::apply_transition_rule(event, src, oi).await +} + +pub(crate) async fn apply_expiry_rule(event: &Event, src: &LcEventSrc, oi: &ScannerObjectInfo) -> bool { + rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::apply_expiry_rule(event, src, oi).await +} + +pub(crate) async fn list_global_tiers() -> Vec { + rustfs_ecstore::api::global::GLOBAL_TierConfigMgr.read().await.list_tiers() +} + +pub(crate) async fn enqueue_global_free_version(oi: ScannerObjectInfo) { + rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState + .write() + .await + .enqueue_free_version(oi) + .await; +} + +pub(crate) async fn enqueue_global_newer_noncurrent( + bucket: &str, + to_delete_objs: Vec, + event: Event, + src: &LcEventSrc, +) -> bool { + rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState + .write() + .await + .enqueue_by_newer_noncurrent(bucket, to_delete_objs, event, src) + .await +} + +pub(crate) async fn queue_replication_heal_internal( + bucket: &str, + oi: ScannerObjectInfo, + rcfg: ReplicationConfig, + retry_count: u32, +) -> ReplicationHealQueueResult { + rustfs_ecstore::api::bucket::replication::queue_replication_heal_internal(bucket, oi, rcfg, retry_count).await +} pub(crate) fn resolve_scanner_object_store_handle() -> Option> { rustfs_ecstore::api::global::resolve_object_store_handle() diff --git a/crates/scanner/tests/common/storage_compat.rs b/crates/scanner/tests/common/storage_compat.rs index 78f8a3b47..44b8332a2 100644 --- a/crates/scanner/tests/common/storage_compat.rs +++ b/crates/scanner/tests/common/storage_compat.rs @@ -14,6 +14,7 @@ #![allow(dead_code, unused_imports)] +use std::collections::HashMap; use std::sync::Arc; use time::OffsetDateTime; @@ -25,21 +26,42 @@ pub(crate) type BucketVersioningSys = rustfs_ecstore::api::bucket::versioning_sy pub(crate) type DiskOption = rustfs_ecstore::api::disk::DiskOption; pub(crate) type ECStore = rustfs_ecstore::api::storage::ECStore; pub(crate) type Endpoint = rustfs_ecstore::api::disk::endpoint::Endpoint; +pub(crate) type EndpointServerPools = rustfs_ecstore::api::layout::EndpointServerPools; pub(crate) type Endpoints = rustfs_ecstore::api::layout::Endpoints; pub(crate) type PoolEndpoints = rustfs_ecstore::api::layout::PoolEndpoints; pub(crate) type ReadCloser = rustfs_ecstore::api::client::transition_api::ReadCloser; pub(crate) type ReaderImpl = rustfs_ecstore::api::client::transition_api::ReaderImpl; pub(crate) type TierConfig = rustfs_ecstore::api::tier::tier_config::TierConfig; +pub(crate) type TierConfigMgr = rustfs_ecstore::api::tier::tier::TierConfigMgr; pub(crate) type TierMinIO = rustfs_ecstore::api::tier::tier_config::TierMinIO; pub(crate) type TierType = rustfs_ecstore::api::tier::tier_config::TierType; pub(crate) type TransitionOptions = rustfs_ecstore::api::bucket::lifecycle::lifecycle::TransitionOptions; pub(crate) type WarmBackendGetOpts = rustfs_ecstore::api::tier::warm_backend::WarmBackendGetOpts; -pub(crate) use rustfs_ecstore::api::disk::DiskAPI; -pub(crate) use rustfs_ecstore::api::global::GLOBAL_TierConfigMgr; -pub(crate) use rustfs_ecstore::api::layout::EndpointServerPools; -pub(crate) use rustfs_ecstore::api::tier::warm_backend::WarmBackend; -pub(crate) use rustfs_ecstore::api::tier::warm_backend::build_transition_put_options; +#[allow(non_snake_case)] +pub(crate) fn EndpointServerPools(pools: Vec) -> EndpointServerPools { + rustfs_ecstore::api::layout::EndpointServerPools::from(pools) +} + +pub(crate) struct GlobalTierConfigMgrCompat; + +#[allow(non_upper_case_globals)] +pub(crate) static GLOBAL_TierConfigMgr: GlobalTierConfigMgrCompat = GlobalTierConfigMgrCompat; + +impl std::ops::Deref for GlobalTierConfigMgrCompat { + type Target = Arc>; + + fn deref(&self) -> &Self::Target { + &rustfs_ecstore::api::global::GLOBAL_TierConfigMgr + } +} + +pub(crate) fn build_transition_put_options( + storage_class: String, + metadata: HashMap, +) -> rustfs_ecstore::api::client::api_put_object::PutObjectOptions { + rustfs_ecstore::api::tier::warm_backend::build_transition_put_options(storage_class, metadata) +} pub(crate) async fn enqueue_transition_for_existing_objects( api: Arc, diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index 384091dbc..2b10f2d6f 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -15,14 +15,16 @@ mod common; use crate::common::storage_compat::{ - BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DiskAPI, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, + BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, GLOBAL_TierConfigMgr, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, TierConfig, TierMinIO, TierType, - TransitionOptions, WarmBackend, WarmBackendGetOpts, build_transition_put_options, enqueue_transition_for_existing_objects, + TransitionOptions, WarmBackendGetOpts, build_transition_put_options, enqueue_transition_for_existing_objects, get_bucket_metadata, init_background_expiry, init_bucket_metadata_sys, init_local_disks, new_disk, path2_bucket_object_with_base_path, update_bucket_metadata, }; use futures::FutureExt; use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT; +use rustfs_ecstore::api::disk::DiskAPI as _; +use rustfs_ecstore::api::tier::warm_backend::WarmBackend; use rustfs_filemeta::FileMeta; use rustfs_scanner::scanner_folder::ScannerItem; use rustfs_scanner::scanner_io::ScannerIODisk; diff --git a/docs/architecture/crate-boundaries.md b/docs/architecture/crate-boundaries.md index 5325b6abd..66420f8d9 100644 --- a/docs/architecture/crate-boundaries.md +++ b/docs/architecture/crate-boundaries.md @@ -125,6 +125,10 @@ directly for `ObjectIO`, `ObjectOperations`, `ListOperations`, `MultipartOperations`, `HealOperations`, and `NamespaceLocking`; ECStore keeps the concrete compatibility traits only for internal implementation and downstream compatibility. +Outer consumers must not import ECStore directly outside compatibility +boundaries except for temporary trait imports needed for method resolution or +local test trait implementations. Non-trait ECStore surfaces must stay behind +local aliases, constants, or wrapper functions. Outer compatibility boundary modules must use `rustfs_ecstore::api` for ECStore public facade surfaces such as layout, storage owner, admin, metrics, diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index e0de87531..d1b9eff53 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,16 +5,17 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-storage-owner-compat-wrappers-main` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086`. -- Stacked on: `origin/main` after API-086 merged. +- Branch: `overtrue/arch-trait-import-compat-cleanup` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088`. +- Stacked on: `origin/main` after API-088 merged. - PR type for this branch: `pure-move` - Runtime behavior changes: none. -- Rust code changes: prune storage-owner compatibility re-exports into local - constants, type aliases, trait imports, and wrapper functions. -- CI/script changes: guard against restoring storage-owner ECStore API - re-exports except temporary trait imports. -- Docs changes: record the API-087 storage-owner compatibility boundary. +- Rust code changes: move remaining ECStore method-resolution trait imports + from compatibility boundaries into direct call sites, and narrow scanner, + heal, and e2e compatibility helpers to local aliases and wrappers. +- CI/script changes: tighten the compatibility re-export guard while allowing + temporary ECStore method-resolution trait imports at call sites. +- Docs changes: record the API-089 trait import compatibility cleanup. ## Phase 0 Tasks @@ -262,6 +263,22 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block - Verification: RustFS compile coverage, admin/app re-export residual scan, migration guard, formatting, diff hygiene, Rust risk scan, pre-commit quality gate, and three-expert review. +- [x] `API-089` Prune trait import compatibility re-exports. + - Completed slice: remove the remaining direct ECStore API `pub use` + compatibility exports from RustFS admin/app/storage and scanner/heal/e2e + boundaries, replacing non-trait access with local wrappers and moving + method-resolution trait imports into the files that call those methods. + - Acceptance: compatibility boundary files no longer expose + `pub(crate) use rustfs_ecstore::api` symbols, while scanner, heal, e2e, + admin, app, and storage call sites keep their existing behavior through + direct trait imports or local wrappers. + - Must preserve: scanner lifecycle and replication evaluation, heal local + disk scanning, e2e RPC signature setup, app restore/lifecycle/object-lock + checks, admin site-replication behavior, storage RPC disk access, and S3 + versioning/replication behavior. + - Verification: RustFS and edge crate compile coverage, compatibility + re-export residual scan, migration guard, formatting, diff hygiene, Rust + risk scan, pre-commit quality gate, and three-expert review. - [x] `G-012` Inventory placement and repair invariants. - Acceptance: [`placement-repair-invariants.md`](placement-repair-invariants.md) records @@ -3295,14 +3312,25 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | API-087 narrows storage-owner compatibility with local aliases/wrappers and an ECStore API re-export guard while leaving only method-resolution trait imports. | -| Migration preservation | passed | Metadata, object-lock, replication proxy metrics, tag/XML helpers, RPC signature checks, tier reload, global accessors, and local disk lookup remain behind existing storage compatibility names. | -| Testing/verification | passed | RustFS compile coverage, storage-owner re-export residual scan, migration guard, formatting, diff hygiene, Rust risk scan, full pre-commit, and three-expert review passed. | +| Quality/architecture | passed | API-089 removes ECStore API re-export compatibility from the remaining app/admin/storage/scanner/heal/e2e boundaries while limiting direct imports to method-resolution traits. | +| Migration preservation | passed | Scanner, heal, e2e, app, admin, and storage consumers keep non-trait ECStore access behind local aliases, wrappers, or proxy statics with existing behavior preserved. | +| Testing/verification | passed | Targeted compile coverage, ECStore API re-export residual scan, migration guard, formatting, diff hygiene, Rust risk scan, full pre-commit, and three-expert review passed. | ## Verification Notes Passed before push: +- Issue #660 API-089 current slice: + - `cargo check -p rustfs -p rustfs-scanner -p rustfs-heal -p e2e_test`: + passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - ECStore API re-export residual scan for compatibility boundaries: passed. + - Rust added-line risk scan on changed Rust files and guard script: passed. + - `make pre-commit`: passed. + - Issue #660 API-087 current slice: - `cargo check -p rustfs`: passed. - `cargo fmt --all --check`: passed. diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index adab64335..1abc2bb81 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -26,10 +26,9 @@ use crate::admin::storage_compat::metadata::{ }; use crate::admin::storage_compat::metadata_sys; use crate::admin::storage_compat::replication::GLOBAL_REPLICATION_STATS; -use crate::admin::storage_compat::replication::{ReplicationConfigurationExt, ResyncOpts, get_global_replication_pool}; +use crate::admin::storage_compat::replication::{ResyncOpts, get_global_replication_pool}; use crate::admin::storage_compat::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials}; use crate::admin::storage_compat::utils::{deserialize, serialize}; -use crate::admin::storage_compat::versioning::VersioningApi; use crate::admin::storage_compat::{delete_admin_config, read_admin_config, save_admin_config}; use crate::admin::storage_compat::{get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port}; use crate::admin::utils::{encode_compatible_admin_payload, read_compatible_admin_body}; @@ -50,6 +49,8 @@ use rustfs_config::{ DEFAULT_CONSOLE_ADDRESS, DEFAULT_DELIMITER, DEFAULT_RUSTFS_TLS_PATH, ENV_RUSTFS_CONSOLE_ADDRESS, ENV_RUSTFS_TLS_PATH, MAX_ADMIN_REQUEST_BODY_SIZE, }; +use rustfs_ecstore::api::bucket::replication::ReplicationConfigurationExt as _; +use rustfs_ecstore::api::bucket::versioning::VersioningApi as _; use rustfs_iam::error::is_err_no_such_service_account; use rustfs_iam::store::{MappedPolicy, UserType}; use rustfs_iam::sys::{ diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 631987a7c..728d3b550 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -25,11 +25,9 @@ use crate::admin::storage_compat::metadata::BUCKET_TARGETS_FILE; use crate::admin::storage_compat::metadata_sys; use crate::admin::storage_compat::read_admin_config_without_migrate; use crate::admin::storage_compat::replication::{ - BucketReplicationResyncStatus, BucketStats, GLOBAL_REPLICATION_STATS, ObjectOpts, ReplicationConfigurationExt, ResyncOpts, - get_global_replication_pool, + BucketReplicationResyncStatus, BucketStats, GLOBAL_REPLICATION_STATS, ObjectOpts, ResyncOpts, get_global_replication_pool, }; use crate::admin::storage_compat::target::{BucketTarget, BucketTargetType, BucketTargets}; -use crate::admin::storage_compat::versioning::VersioningApi; use crate::admin::storage_compat::versioning_sys::BucketVersioningSys; use crate::admin::storage_compat::{get_global_bucket_monitor, get_global_deployment_id, get_global_region}; use crate::app::context::resolve_object_store_handle; @@ -62,6 +60,8 @@ use rustfs_config::{ ENABLE_KEY, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_SKIP_TLS_VERIFY, }; +use rustfs_ecstore::api::bucket::replication::ReplicationConfigurationExt as _; +use rustfs_ecstore::api::bucket::versioning::VersioningApi as _; use rustfs_filemeta::{ReplicationStatusType, ReplicationType}; use rustfs_madmin::utils::parse_duration; use rustfs_notify::{Event as NotificationEvent, notification_system}; diff --git a/rustfs/src/admin/storage_compat.rs b/rustfs/src/admin/storage_compat.rs index 184b33742..e12f1ed94 100644 --- a/rustfs/src/admin/storage_compat.rs +++ b/rustfs/src/admin/storage_compat.rs @@ -196,8 +196,6 @@ pub(crate) mod quota { pub(crate) mod replication { use std::sync::Arc; - pub(crate) use rustfs_ecstore::api::bucket::replication::ReplicationConfigurationExt; - pub(crate) type BucketReplicationResyncStatus = rustfs_ecstore::api::bucket::replication::BucketReplicationResyncStatus; pub(crate) type BucketStats = rustfs_ecstore::api::bucket::replication::BucketStats; pub(crate) type DynReplicationPool = rustfs_ecstore::api::bucket::replication::DynReplicationPool; @@ -250,9 +248,7 @@ pub(crate) mod utils { } } -pub(crate) mod versioning { - pub(crate) use rustfs_ecstore::api::bucket::versioning::VersioningApi; -} +pub(crate) mod versioning {} pub(crate) mod versioning_sys { pub(crate) type BucketVersioningSys = rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys; diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 263403ccf..db3f98980 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -27,7 +27,8 @@ use crate::app::storage_compat::object_api_utils::to_s3s_etag; use crate::app::storage_compat::{ bucket_target_sys::BucketTargetSys, lifecycle::bucket_lifecycle_ops::{ - enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, validate_transition_tier, + enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, validate_lifecycle_config, + validate_transition_tier, }, metadata::{ BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, @@ -35,11 +36,9 @@ use crate::app::storage_compat::{ BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, }, metadata_sys, - object_lock::ObjectLockApi, policy_sys::PolicySys, target::{BucketTargetType, BucketTargets}, utils::serialize, - versioning::VersioningApi, versioning_sys::BucketVersioningSys, }; use crate::auth::get_condition_values_with_client_info; @@ -58,6 +57,8 @@ use futures::StreamExt; use http::StatusCode; use metrics::counter; use rustfs_config::RUSTFS_REGION; +use rustfs_ecstore::api::bucket::object_lock::ObjectLockApi as _; +use rustfs_ecstore::api::bucket::versioning::VersioningApi as _; use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta}; use rustfs_policy::policy::{ action::{Action, S3Action}, @@ -1631,7 +1632,7 @@ impl DefaultBucketUsecase { } }; - if let Err(err) = crate::app::storage_compat::lifecycle::lifecycle::Lifecycle::validate(&input_cfg, &rcfg).await { + if let Err(err) = validate_lifecycle_config(&input_cfg, &rcfg).await { return Err(s3_error!(InvalidArgument, "{err}")); } diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index 3908c2752..3d5ccd361 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -15,7 +15,7 @@ use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase}; use crate::app::bucket_usecase::DefaultBucketUsecase; use crate::app::storage_compat::{ - ECStore, Endpoint, EndpointServerPools, Endpoints, GLOBAL_TierConfigMgr, PoolEndpoints, TierConfig, TierType, WarmBackend, + ECStore, Endpoint, EndpointServerPools, Endpoints, GLOBAL_TierConfigMgr, PoolEndpoints, TierConfig, TierType, WarmBackendGetOpts, metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG}, metadata_sys, @@ -31,6 +31,7 @@ use futures::FutureExt; use futures::stream; use http::{Extensions, HeaderMap, HeaderValue, Method, Uri, header::IF_NONE_MATCH}; use rustfs_config::{ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT}; +use rustfs_ecstore::api::tier::warm_backend::WarmBackend; use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager}; use rustfs_storage_api::{ BucketOperations, BucketOptions, ListOperations as _, MakeBucketOptions, MultipartOperations as _, ObjectIO as _, diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 99d1ad3ff..49de2af3b 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -62,8 +62,8 @@ use crate::app::storage_compat::{get_lock_acquire_timeout, is_valid_storage_clas use crate::app::storage_compat::{ lifecycle::{ bucket_lifecycle_audit::LcEventSrc, - bucket_lifecycle_ops::{RestoreRequestOps, enqueue_transition_immediate, post_restore_opts}, - lifecycle::{self, Lifecycle, TransitionOptions}, + bucket_lifecycle_ops::{enqueue_transition_immediate, post_restore_opts}, + lifecycle::{self, TransitionOptions}, }, metadata_sys, object_lock::{ @@ -72,15 +72,18 @@ use crate::app::storage_compat::{ }, quota::QuotaOperation, replication::{ - DeletedObjectReplicationInfo, ObjectOpts as ReplicationObjectOpts, ReplicationConfigurationExt, check_replicate_delete, - get_must_replicate_options, must_replicate, schedule_replication, schedule_replication_delete, + DeletedObjectReplicationInfo, ObjectOpts as ReplicationObjectOpts, check_replicate_delete, get_must_replicate_options, + must_replicate, schedule_replication, schedule_replication_delete, }, tagging::decode_tags, - versioning::VersioningApi, versioning_sys::BucketVersioningSys, }; use crate::server::convert_ecstore_object_info; use rustfs_concurrency::GetObjectQueueSnapshot; +use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::RestoreRequestOps as _; +use rustfs_ecstore::api::bucket::lifecycle::lifecycle::Lifecycle as _; +use rustfs_ecstore::api::bucket::replication::ReplicationConfigurationExt as _; +use rustfs_ecstore::api::bucket::versioning::VersioningApi as _; use rustfs_filemeta::{ REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType, ReplicationType, RestoreStatusOps, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map, diff --git a/rustfs/src/app/storage_compat.rs b/rustfs/src/app/storage_compat.rs index 05572c996..a57b10efe 100644 --- a/rustfs/src/app/storage_compat.rs +++ b/rustfs/src/app/storage_compat.rs @@ -47,8 +47,6 @@ pub(crate) type TierConfig = rustfs_ecstore::api::tier::tier_config::TierConfig; #[cfg(test)] pub(crate) type TierType = rustfs_ecstore::api::tier::tier_config::TierType; #[cfg(test)] -pub(crate) use rustfs_ecstore::api::tier::warm_backend::WarmBackend; -#[cfg(test)] pub(crate) type WarmBackendGetOpts = rustfs_ecstore::api::tier::warm_backend::WarmBackendGetOpts; #[cfg(test)] @@ -79,8 +77,6 @@ pub(crate) mod lifecycle { use super::ECStore; use super::bucket_lifecycle_audit::LcEventSrc; - pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::RestoreRequestOps; - pub(crate) type ExpiryState = rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::ExpiryState; pub(crate) struct GlobalExpiryStateCompat; @@ -131,11 +127,18 @@ pub(crate) mod lifecycle { pub(crate) async fn validate_transition_tier(lc: &s3s::dto::BucketLifecycleConfiguration) -> Result<(), std::io::Error> { rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier(lc).await } + + pub(crate) async fn validate_lifecycle_config( + lc: &s3s::dto::BucketLifecycleConfiguration, + lock_config: &s3s::dto::ObjectLockConfiguration, + ) -> Result<(), std::io::Error> { + use rustfs_ecstore::api::bucket::lifecycle::lifecycle::Lifecycle as _; + + lc.validate(lock_config).await + } } pub(crate) mod lifecycle_contract { - pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::Lifecycle; - #[cfg(test)] pub(crate) type IlmAction = rustfs_ecstore::api::bucket::lifecycle::lifecycle::IlmAction; pub(crate) type Event = rustfs_ecstore::api::bucket::lifecycle::lifecycle::Event; @@ -330,8 +333,6 @@ pub(crate) mod object_lock { rustfs_ecstore::api::bucket::object_lock::objectlock_sys::is_retention_active(mode, retain_until_date) } } - - pub(crate) use rustfs_ecstore::api::bucket::object_lock::ObjectLockApi; } pub(crate) mod policy_sys { @@ -350,8 +351,6 @@ pub(crate) mod replication { use std::collections::HashMap; use std::sync::Arc; - pub(crate) use rustfs_ecstore::api::bucket::replication::ReplicationConfigurationExt; - pub(crate) type DeletedObjectReplicationInfo = rustfs_ecstore::api::bucket::replication::DeletedObjectReplicationInfo; pub(crate) type MustReplicateOptions = rustfs_ecstore::api::bucket::replication::MustReplicateOptions; pub(crate) type ObjectOpts = rustfs_ecstore::api::bucket::replication::ObjectOpts; @@ -414,9 +413,7 @@ pub(crate) mod utils { } } -pub(crate) mod versioning { - pub(crate) use rustfs_ecstore::api::bucket::versioning::VersioningApi; -} +pub(crate) mod versioning {} pub(crate) mod versioning_sys { pub(crate) type BucketVersioningSys = rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys; diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 2efabac45..9a6ffdbf2 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -23,16 +23,18 @@ use crate::storage::options::get_opts; use crate::storage::s3_api::acl; use crate::storage::storage_compat::{ BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG, - BUCKET_WEBSITE_CONFIG, BucketVersioningSys, OBJECT_LOCK_CONFIG, ReplicationConfigurationExt, StorageError, VersioningApi, - check_retention_for_modification, decode_tags, decode_tags_to_map, delete_bucket_metadata_config, encode_tags, - get_bucket_accelerate_config, get_bucket_logging_config, get_bucket_object_lock_config, get_bucket_replication_config, - get_bucket_request_payment_config, get_bucket_website_config, is_err_bucket_not_found, is_err_object_not_found, - is_err_version_not_found, record_replication_proxy, serialize, update_bucket_metadata_config, + BUCKET_WEBSITE_CONFIG, BucketVersioningSys, OBJECT_LOCK_CONFIG, StorageError, check_retention_for_modification, decode_tags, + decode_tags_to_map, delete_bucket_metadata_config, encode_tags, get_bucket_accelerate_config, get_bucket_logging_config, + get_bucket_object_lock_config, get_bucket_replication_config, get_bucket_request_payment_config, get_bucket_website_config, + is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, record_replication_proxy, serialize, + update_bucket_metadata_config, }; use crate::storage::{parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled}; use crate::table_catalog; use http::StatusCode; use metrics::{counter, histogram}; +use rustfs_ecstore::api::bucket::replication::ReplicationConfigurationExt as _; +use rustfs_ecstore::api::bucket::versioning::VersioningApi as _; use rustfs_io_metrics::record_s3_op; use rustfs_s3_ops::S3Operation; use rustfs_storage_api::{BucketOperations, BucketOptions, ObjectLockRetentionOptions, ObjectOperations as _}; diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index 826bdcf58..49038546c 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -17,12 +17,13 @@ use crate::error::ApiError; use crate::server::cors; use crate::storage::ecfs::ListObjectUnorderedQuery; use crate::storage::storage_compat::{ - ReplicationConfigurationExt, StorageError, add_object_lock_years, get_bucket_cors_config, get_bucket_object_lock_config, - get_bucket_replication_config, resolve_object_store_handle, + StorageError, add_object_lock_years, get_bucket_cors_config, get_bucket_object_lock_config, get_bucket_replication_config, + resolve_object_store_handle, }; use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE}; use http::{HeaderMap, HeaderValue, StatusCode}; use metrics::counter; +use rustfs_ecstore::api::bucket::replication::ReplicationConfigurationExt as _; use rustfs_storage_api::{BucketOperations, BucketOptions}; use rustfs_targets::EventName; use rustfs_targets::arn::{TargetID, TargetIDError}; diff --git a/rustfs/src/storage/rpc/bucket.rs b/rustfs/src/storage/rpc/bucket.rs index 6083157ae..ee868d5fe 100644 --- a/rustfs/src/storage/rpc/bucket.rs +++ b/rustfs/src/storage/rpc/bucket.rs @@ -13,6 +13,7 @@ // limitations under the License. use super::*; +use rustfs_ecstore::api::rpc::PeerS3Client as _; impl NodeService { pub(super) async fn handle_delete_bucket_metadata( diff --git a/rustfs/src/storage/rpc/http_service.rs b/rustfs/src/storage/rpc/http_service.rs index 47b18d2c0..18bc92167 100644 --- a/rustfs/src/storage/rpc/http_service.rs +++ b/rustfs/src/storage/rpc/http_service.rs @@ -15,15 +15,16 @@ use crate::server::RPC_PREFIX; use crate::storage::request_context::spawn_traced; use crate::storage::storage_compat::DEFAULT_READ_BUFFER_SIZE; +use crate::storage::storage_compat::WalkDirOptions; use crate::storage::storage_compat::find_local_disk_by_ref; use crate::storage::storage_compat::verify_rpc_signature; -use crate::storage::storage_compat::{DiskAPI, WalkDirOptions}; use bytes::{Bytes, BytesMut}; use futures_util::TryStreamExt; use http::{HeaderMap, Method, Request, Response, StatusCode, Uri}; use http_body_util::{BodyExt, Limited}; use hyper::body::Incoming; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; +use rustfs_ecstore::api::disk::DiskAPI as _; use rustfs_io_metrics::internode_metrics::{ INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics, diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index ac9e459a8..b430e8c78 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -17,17 +17,18 @@ use crate::admin::service::{ site_replication::reload_site_replication_runtime_state, }; use crate::storage::storage_compat::{ - CollectMetricsOpts, DeleteOptions, DiskAPI, DiskError, DiskInfoOptions, DiskStore, FileInfoVersions, LocalPeerS3Client, - MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, ReadMultipleReq, ReadMultipleResp, ReadOptions, - SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, - find_local_disk_by_ref, get_global_lock_client, get_local_server_property, load_bucket_metadata, - reload_transition_tier_config, resolve_object_store_handle, set_bucket_metadata, + CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, FileInfoVersions, LocalPeerS3Client, MetricType, + PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, + SERVICE_SIGNAL_RELOAD_DYNAMIC, UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, + get_global_lock_client, get_local_server_property, load_bucket_metadata, reload_transition_tier_config, + resolve_object_store_handle, set_bucket_metadata, }; use bytes::Bytes; use futures::Stream; use futures_util::future::join_all; use rmp_serde::Deserializer; use rustfs_common::{get_global_local_node_name, heal_channel::HealOpts}; +use rustfs_ecstore::api::disk::DiskAPI as _; use rustfs_filemeta::{FileInfo, MetacacheReader}; use rustfs_iam::{get_global_iam_sys, store::UserType}; use rustfs_lock::{LockClient, LockRequest}; diff --git a/rustfs/src/storage/storage_compat.rs b/rustfs/src/storage/storage_compat.rs index 83be0c390..b5514e657 100644 --- a/rustfs/src/storage/storage_compat.rs +++ b/rustfs/src/storage/storage_compat.rs @@ -14,11 +14,6 @@ use std::sync::Arc; -pub(crate) use rustfs_ecstore::api::bucket::replication::ReplicationConfigurationExt; -pub(crate) use rustfs_ecstore::api::bucket::versioning::VersioningApi; -pub(crate) use rustfs_ecstore::api::disk::DiskAPI; -pub(crate) use rustfs_ecstore::api::rpc::PeerS3Client; - pub(crate) const BUCKET_ACCELERATE_CONFIG: &str = rustfs_ecstore::api::bucket::metadata::BUCKET_ACCELERATE_CONFIG; pub(crate) const BUCKET_LOGGING_CONFIG: &str = rustfs_ecstore::api::bucket::metadata::BUCKET_LOGGING_CONFIG; pub(crate) const BUCKET_REQUEST_PAYMENT_CONFIG: &str = rustfs_ecstore::api::bucket::metadata::BUCKET_REQUEST_PAYMENT_CONFIG; diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 57ce16b50..a23084861 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -684,7 +684,18 @@ fi --glob '!crates/ecstore/**' \ --glob '!**/storage_compat.rs' \ --glob '!target/**' || true -) >"$DIRECT_ECSTORE_IMPORT_HITS_FILE" +) | + perl -ne ' + next if /\buse rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::RestoreRequestOps(?:\s+as\s+_)?;/; + next if /\buse rustfs_ecstore::api::bucket::lifecycle::lifecycle::Lifecycle(?:\s+as\s+_)?;/; + next if /\buse rustfs_ecstore::api::bucket::object_lock::ObjectLockApi(?:\s+as\s+_)?;/; + next if /\buse rustfs_ecstore::api::bucket::replication::ReplicationConfigurationExt(?:\s+as\s+_)?;/; + next if /\buse rustfs_ecstore::api::bucket::versioning::VersioningApi(?:\s+as\s+_)?;/; + next if /\buse rustfs_ecstore::api::disk::DiskAPI(?:\s+as\s+_)?;/; + next if /\buse rustfs_ecstore::api::rpc::PeerS3Client(?:\s+as\s+_)?;/; + next if /\buse rustfs_ecstore::api::tier::warm_backend::WarmBackend(?:\s+as\s+_)?;/; + print; + ' >"$DIRECT_ECSTORE_IMPORT_HITS_FILE" if [[ -s "$DIRECT_ECSTORE_IMPORT_HITS_FILE" ]]; then report_failure "direct rustfs_ecstore imports outside compatibility boundaries are forbidden: $(paste -sd '; ' "$DIRECT_ECSTORE_IMPORT_HITS_FILE")"