diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 58884b774..a70f71b83 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -37,7 +37,7 @@ use crate::global::GLOBAL_LocalNodeName; use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id}; use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks}; use crate::store::ECStore; -use crate::store_api::{GetObjectReader, MultipartOperations, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete}; +use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, ObjectToDelete}; use crate::tier::warm_backend::WarmBackendGetOpts; use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded}; use futures::Future; @@ -58,7 +58,7 @@ use rustfs_filemeta::{ VersionPurgeStatusType, get_file_info, is_restored_object_on_disk, }; use rustfs_s3_types::EventName; -use rustfs_storage_api::{HTTPRangeSpec, ListOperations as _}; +use rustfs_storage_api::{HTTPRangeSpec, ListOperations as _, MultipartOperations as _, ObjectOperations as _}; use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold}; use s3s::dto::{ BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest, @@ -2870,12 +2870,12 @@ mod tests { use crate::error::is_err_invalid_upload_id; use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY}; use crate::store::ECStore; - use crate::store_api::{MultipartOperations, ObjectInfo, ObjectOptions, PutObjReader}; + use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader}; use futures::FutureExt; use rustfs_common::metrics::{IlmAction, global_metrics}; use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX; use rustfs_filemeta::{ReplicateDecision, VersionPurgeStatusType}; - use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions}; + use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations as _}; use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, Timestamp}; use serial_test::serial; use sha2::{Digest, Sha256}; diff --git a/crates/ecstore/src/client/object_handlers_common.rs b/crates/ecstore/src/client/object_handlers_common.rs index c4aa96272..e840ac7fc 100644 --- a/crates/ecstore/src/client/object_handlers_common.rs +++ b/crates/ecstore/src/client/object_handlers_common.rs @@ -25,9 +25,10 @@ use crate::bucket::replication::{DeletedObjectReplicationInfo, check_replicate_d use crate::bucket::versioning::VersioningApi; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::store::ECStore; -use crate::store_api::{ObjectOperations, ObjectOptions, ObjectToDelete}; +use crate::store_api::{ObjectOptions, ObjectToDelete}; use rustfs_filemeta::{REPLICATE_INCOMING_DELETE, ReplicationState, version_purge_statuses_map}; use rustfs_lock::MAX_DELETE_LIST; +use rustfs_storage_api::ObjectOperations as _; fn lifecycle_version_delete_replication_state( replicate_decision_str: String, diff --git a/crates/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs index c19eea58d..dadaa2de9 100644 --- a/crates/ecstore/src/config/com.rs +++ b/crates/ecstore/src/config/com.rs @@ -1217,7 +1217,7 @@ mod tests { use crate::error::{Error, Result}; use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type}; use crate::set_disk::SetDisks; - use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader}; + use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectInfo, ObjectOptions, PutObjReader}; use http::HeaderMap; use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS}; use rustfs_config::notify::{ @@ -1408,7 +1408,15 @@ mod tests { } #[async_trait::async_trait] - impl ObjectIO for LockingConfigStorage { + impl rustfs_storage_api::ObjectIO for LockingConfigStorage { + type Error = Error; + type RangeSpec = HTTPRangeSpec; + type HeaderMap = HeaderMap; + type ObjectOptions = ObjectOptions; + type ObjectInfo = ObjectInfo; + type GetObjectReader = GetObjectReader; + type PutObjectReader = PutObjReader; + async fn get_object_reader( &self, bucket: &str, diff --git a/crates/ecstore/src/data_movement.rs b/crates/ecstore/src/data_movement.rs index 83b50e8a6..508bfa68a 100644 --- a/crates/ecstore/src/data_movement.rs +++ b/crates/ecstore/src/data_movement.rs @@ -14,12 +14,10 @@ use crate::error::{Error, Result, is_err_data_movement_overwrite, is_err_object_not_found, is_err_version_not_found}; use crate::store::ECStore; -use crate::store_api::{ - GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, PutObjReader, -}; +use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; use bytes::Bytes; use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex}; -use rustfs_storage_api::CompletePart; +use rustfs_storage_api::{CompletePart, MultipartOperations as _, ObjectIO as _, ObjectOperations as _}; use rustfs_utils::path::encode_dir_object; use std::io::Cursor; use std::pin::Pin; diff --git a/crates/ecstore/src/data_usage.rs b/crates/ecstore/src/data_usage.rs index 503b9eb4c..03f2bccd0 100644 --- a/crates/ecstore/src/data_usage.rs +++ b/crates/ecstore/src/data_usage.rs @@ -30,7 +30,7 @@ use rustfs_data_usage::{ BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary, }; use rustfs_io_metrics::record_system_path_failure; -use rustfs_storage_api::ListOperations as _; +use rustfs_storage_api::{ListOperations as _, ObjectIO as _}; use rustfs_utils::path::SLASH_SEPARATOR; use std::{ collections::{HashMap, HashSet, hash_map::Entry}, @@ -715,7 +715,7 @@ pub fn cache_to_data_usage_info(cache: &DataUsageCache, path: &str, buckets: &[r // Helper functions for DataUsageCache operations pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str) -> crate::error::Result { use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; - use crate::store_api::{ObjectIO, ObjectOptions}; + use crate::store_api::ObjectOptions; use http::HeaderMap; use rand::RngExt; use std::path::Path; diff --git a/crates/ecstore/src/pools.rs b/crates/ecstore/src/pools.rs index c0b31bc31..32ac2190f 100644 --- a/crates/ecstore/src/pools.rs +++ b/crates/ecstore/src/pools.rs @@ -38,7 +38,7 @@ use crate::error::{ use crate::notification_sys::get_global_notification_sys; use crate::resolve_object_store_handle; use crate::set_disk::SetDisks; -use crate::store_api::{GetObjectReader, HealOperations, ObjectIO, ObjectOperations, ObjectOptions}; +use crate::store_api::{GetObjectReader, HealOperations, ObjectOptions}; use crate::{global::GLOBAL_LifecycleSys, sets::Sets, store::ECStore}; use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered}; @@ -50,7 +50,9 @@ use rustfs_common::defer; use rustfs_common::heal_channel::HealOpts; use rustfs_concurrency::workers::Workers; use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; -use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions, StorageAdminApi}; +use rustfs_storage_api::{ + BucketOperations, BucketOptions, MakeBucketOptions, ObjectIO as _, ObjectOperations as _, StorageAdminApi, +}; use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path}; use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration}; use serde::{Deserialize, Serialize}; diff --git a/crates/ecstore/src/rebalance.rs b/crates/ecstore/src/rebalance.rs index ab90da22b..e142725a7 100644 --- a/crates/ecstore/src/rebalance.rs +++ b/crates/ecstore/src/rebalance.rs @@ -23,11 +23,11 @@ use crate::global::get_global_endpoints; use crate::pools::ListCallback; use crate::set_disk::{SetDisks, get_lock_acquire_timeout}; use crate::store::ECStore; -use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions}; +use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOptions}; use http::HeaderMap; use rand::RngExt as _; use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; -use rustfs_storage_api::{HTTPRangeSpec, StorageAdminApi}; +use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _, StorageAdminApi}; use rustfs_utils::path::encode_dir_object; use serde::{Deserialize, Serialize}; use std::collections::HashSet; diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 11657a479..8fc2654bc 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -90,6 +90,7 @@ use rustfs_storage_api::{ BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, PartInfo, }; +use rustfs_storage_api::{MultipartOperations as _, ObjectIO as _, ObjectOperations as _}; use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING; use rustfs_utils::http::headers::AMZ_STORAGE_CLASS; use rustfs_utils::http::headers::{ @@ -828,7 +829,15 @@ fn classify_small_write_path(is_inline_buffer: bool, object_size: i64, block_siz } #[async_trait::async_trait] -impl ObjectIO for SetDisks { +impl rustfs_storage_api::ObjectIO for SetDisks { + type Error = Error; + type RangeSpec = HTTPRangeSpec; + type HeaderMap = HeaderMap; + type ObjectOptions = ObjectOptions; + type ObjectInfo = ObjectInfo; + type GetObjectReader = GetObjectReader; + type PutObjectReader = PutObjReader; + #[tracing::instrument(level = "debug", skip(self))] async fn get_object_reader( &self, @@ -1791,7 +1800,14 @@ fn check_object_lock_retention_update(bucket: &str, object: &str, obj_info: &Obj } #[async_trait::async_trait] -impl ObjectOperations for SetDisks { +impl rustfs_storage_api::ObjectOperations for SetDisks { + type Error = Error; + type ObjectInfo = ObjectInfo; + type ObjectOptions = ObjectOptions; + type FileInfo = FileInfo; + type ObjectToDelete = ObjectToDelete; + type DeletedObject = DeletedObject; + #[tracing::instrument(skip(self))] async fn copy_object( &self, @@ -2866,7 +2882,8 @@ impl ObjectOperations for SetDisks { #[tracing::instrument(skip(self))] async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { - let get_object_reader = ::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?; + let get_object_reader = + ::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?; // Stream to sink to avoid loading entire object into memory during verification let mut reader = get_object_reader.stream; tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?; @@ -3035,7 +3052,18 @@ impl rustfs_storage_api::ListOperations for SetDisks { } #[async_trait::async_trait] -impl MultipartOperations for SetDisks { +impl rustfs_storage_api::MultipartOperations for SetDisks { + type Error = Error; + type ObjectInfo = ObjectInfo; + type ObjectOptions = ObjectOptions; + type PutObjectReader = PutObjReader; + type CompletePart = CompletePart; + type ListMultipartsInfo = ListMultipartsInfo; + type MultipartUploadResult = MultipartUploadResult; + type PartInfo = PartInfo; + type MultipartInfo = MultipartInfo; + type ListPartsInfo = ListPartsInfo; + #[tracing::instrument(skip(self))] async fn copy_object_part( &self, @@ -4991,7 +5019,7 @@ mod tests { use rustfs_filemeta::ReplicationState; use rustfs_lock::client::local::LocalClient; use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats}; - use rustfs_storage_api::CompletePart; + use rustfs_storage_api::{CompletePart, ObjectOperations as _}; use serial_test::serial; use std::collections::HashMap; use tempfile::TempDir; diff --git a/crates/ecstore/src/sets.rs b/crates/ecstore/src/sets.rs index 6399b5093..545a4a7a9 100644 --- a/crates/ecstore/src/sets.rs +++ b/crates/ecstore/src/sets.rs @@ -28,8 +28,8 @@ use crate::{ global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure}, set_disk::SetDisks, store_api::{ - DeletedObject, GetObjectReader, HealOperations, ListObjectVersionsInfo, ListObjectsV2Info, MultipartOperations, - NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader, + DeletedObject, GetObjectReader, HealOperations, ListObjectVersionsInfo, ListObjectsV2Info, NamespaceLocking, ObjectInfo, + ObjectOptions, ObjectToDelete, PutObjReader, }, store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file}, }; @@ -53,6 +53,7 @@ use rustfs_storage_api::{ BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, PartInfo, }; +use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _}; use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash}; use std::{collections::HashMap, sync::Arc}; use tokio::sync::RwLock; @@ -402,7 +403,15 @@ fn apply_delete_objects_results( } #[async_trait::async_trait] -impl ObjectIO for Sets { +impl rustfs_storage_api::ObjectIO for Sets { + type Error = Error; + type RangeSpec = HTTPRangeSpec; + type HeaderMap = HeaderMap; + type ObjectOptions = ObjectOptions; + type ObjectInfo = ObjectInfo; + type GetObjectReader = GetObjectReader; + type PutObjectReader = PutObjReader; + #[tracing::instrument(level = "debug", skip(self, object, h, opts))] async fn get_object_reader( &self, @@ -447,7 +456,14 @@ impl BucketOperations for Sets { } #[async_trait::async_trait] -impl ObjectOperations for Sets { +impl rustfs_storage_api::ObjectOperations for Sets { + type Error = Error; + type ObjectInfo = ObjectInfo; + type ObjectOptions = ObjectOptions; + type FileInfo = FileInfo; + type ObjectToDelete = ObjectToDelete; + type DeletedObject = DeletedObject; + async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { self.get_disks_by_key(object).get_object_info(bucket, object, opts).await } @@ -696,7 +712,18 @@ impl rustfs_storage_api::ListOperations for Sets { } #[async_trait::async_trait] -impl MultipartOperations for Sets { +impl rustfs_storage_api::MultipartOperations for Sets { + type Error = Error; + type ObjectInfo = ObjectInfo; + type ObjectOptions = ObjectOptions; + type PutObjectReader = PutObjReader; + type CompletePart = CompletePart; + type ListMultipartsInfo = ListMultipartsInfo; + type MultipartUploadResult = MultipartUploadResult; + type PartInfo = PartInfo; + type MultipartInfo = MultipartInfo; + type ListPartsInfo = ListPartsInfo; + #[tracing::instrument(skip(self))] async fn list_multipart_uploads( &self, diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index 66eee815a..0355ef753 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -53,7 +53,7 @@ use crate::notification_sys::get_global_notification_sys; use crate::pools::PoolMeta; use crate::rebalance::RebalanceMeta; use crate::rpc::RemoteClient; -use crate::store_api::{ListObjectVersionsInfo, ObjectIO, ObjectInfoOrErr, WalkOptions}; +use crate::store_api::{ListObjectVersionsInfo, ObjectInfoOrErr, WalkOptions}; use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config}; use crate::tier::tier::TierConfigMgr; use crate::{ @@ -63,8 +63,8 @@ use crate::{ rpc::S3PeerSys, sets::Sets, store_api::{ - DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, MultipartOperations, NamespaceLocking, ObjectInfo, - ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader, + DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, NamespaceLocking, ObjectInfo, ObjectOptions, + ObjectToDelete, PutObjReader, }, store_init, }; @@ -367,7 +367,15 @@ impl Clone for PoolObjInfo { // } #[async_trait::async_trait] -impl ObjectIO for ECStore { +impl rustfs_storage_api::ObjectIO for ECStore { + type Error = Error; + type RangeSpec = HTTPRangeSpec; + type HeaderMap = HeaderMap; + type ObjectOptions = ObjectOptions; + type ObjectInfo = ObjectInfo; + type GetObjectReader = GetObjectReader; + type PutObjectReader = PutObjReader; + #[instrument(level = "debug", skip(self))] async fn get_object_reader( &self, @@ -420,7 +428,14 @@ impl BucketOperations for ECStore { } #[async_trait::async_trait] -impl ObjectOperations for ECStore { +impl rustfs_storage_api::ObjectOperations for ECStore { + type Error = Error; + type ObjectInfo = ObjectInfo; + type ObjectOptions = ObjectOptions; + type FileInfo = FileInfo; + type ObjectToDelete = ObjectToDelete; + type DeletedObject = DeletedObject; + async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { self.handle_get_object_info(bucket, object, opts).await } @@ -569,7 +584,18 @@ impl rustfs_storage_api::ListOperations for ECStore { } #[async_trait::async_trait] -impl MultipartOperations for ECStore { +impl rustfs_storage_api::MultipartOperations for ECStore { + type Error = Error; + type ObjectInfo = ObjectInfo; + type ObjectOptions = ObjectOptions; + type PutObjectReader = PutObjReader; + type CompletePart = CompletePart; + type ListMultipartsInfo = ListMultipartsInfo; + type MultipartUploadResult = MultipartUploadResult; + type PartInfo = PartInfo; + type MultipartInfo = MultipartInfo; + type ListPartsInfo = ListPartsInfo; + #[instrument(skip(self))] async fn list_multipart_uploads( &self, diff --git a/crates/ecstore/src/store/multipart.rs b/crates/ecstore/src/store/multipart.rs index ba36225fa..a39a8324b 100644 --- a/crates/ecstore/src/store/multipart.rs +++ b/crates/ecstore/src/store/multipart.rs @@ -13,6 +13,7 @@ // limitations under the License. use super::*; +use rustfs_storage_api::MultipartOperations as _; impl ECStore { #[instrument(skip(self))] diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 2ddb33436..d84017cf5 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -21,6 +21,7 @@ use rustfs_io_metrics::{ record_object_lock_diag_acquire_duration, record_object_lock_diag_hold_duration, record_object_lock_diag_slow_acquire, record_object_lock_diag_slow_hold, }; +use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _}; use std::{ fmt, pin::Pin, @@ -1254,7 +1255,8 @@ impl ECStore { } pub(super) async fn handle_verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { - let get_object_reader = ::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?; + let get_object_reader = + ::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?; // Stream to sink to avoid loading entire object into memory during verification let mut reader = get_object_reader.stream; tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?; diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index 65475e924..c16bc75a4 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -14,7 +14,7 @@ use super::*; use crate::config::get_global_storage_class; -use rustfs_storage_api::StorageAdminApi; +use rustfs_storage_api::{ObjectOperations as _, StorageAdminApi}; struct LatestObjectInfoCandidate { info: Option, diff --git a/crates/ecstore/src/store_api/traits.rs b/crates/ecstore/src/store_api/traits.rs index 1b38458bd..854ce79bc 100644 --- a/crates/ecstore/src/store_api/traits.rs +++ b/crates/ecstore/src/store_api/traits.rs @@ -1,51 +1,65 @@ use super::*; use rustfs_storage_api::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo}; -#[async_trait::async_trait] -pub trait ObjectIO: Send + Sync + Debug + 'static { - async fn get_object_reader( - &self, - bucket: &str, - object: &str, - range: Option, - h: HeaderMap, - opts: &ObjectOptions, - ) -> Result; +pub trait ObjectIO: + rustfs_storage_api::ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + > + Send + + Sync + + Debug + + 'static +{ +} - async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result; +impl ObjectIO for T where + T: rustfs_storage_api::ObjectIO< + Error = Error, + RangeSpec = HTTPRangeSpec, + HeaderMap = HeaderMap, + ObjectOptions = ObjectOptions, + ObjectInfo = ObjectInfo, + GetObjectReader = GetObjectReader, + PutObjectReader = PutObjReader, + > + Send + + Sync + + Debug + + 'static +{ } /// Object-level storage operations (beyond basic I/O in ObjectIO). -#[async_trait::async_trait] -#[allow(clippy::too_many_arguments)] -pub trait ObjectOperations: Send + Sync + Debug { - async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; - async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>; - async fn copy_object( - &self, - src_bucket: &str, - src_object: &str, - dst_bucket: &str, - dst_object: &str, - src_info: &mut ObjectInfo, - src_opts: &ObjectOptions, - dst_opts: &ObjectOptions, - ) -> Result; - async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()>; - async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result; - async fn delete_objects( - &self, - bucket: &str, - objects: Vec, - opts: ObjectOptions, - ) -> (Vec, Vec>); - async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; - async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; - async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result; - async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; - async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()>; - async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>; - async fn restore_transitioned_object(self: Arc, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>; +pub trait ObjectOperations: + rustfs_storage_api::ObjectOperations< + Error = Error, + ObjectInfo = ObjectInfo, + ObjectOptions = ObjectOptions, + FileInfo = FileInfo, + ObjectToDelete = ObjectToDelete, + DeletedObject = DeletedObject, + > + Send + + Sync + + Debug +{ +} + +impl ObjectOperations for T where + T: rustfs_storage_api::ObjectOperations< + Error = Error, + ObjectInfo = ObjectInfo, + ObjectOptions = ObjectOptions, + FileInfo = FileInfo, + ObjectToDelete = ObjectToDelete, + DeletedObject = DeletedObject, + > + Send + + Sync + + Debug +{ } /// Listing and walking operations. @@ -80,67 +94,40 @@ impl ListOperations for T where } /// Multipart upload operations. -#[async_trait::async_trait] -#[allow(clippy::too_many_arguments)] -pub trait MultipartOperations: Send + Sync + Debug { - async fn list_multipart_uploads( - &self, - bucket: &str, - prefix: &str, - key_marker: Option, - upload_id_marker: Option, - delimiter: Option, - max_uploads: usize, - ) -> Result; - async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; - async fn copy_object_part( - &self, - src_bucket: &str, - src_object: &str, - dst_bucket: &str, - dst_object: &str, - upload_id: &str, - part_id: usize, - start_offset: i64, - length: i64, - src_info: &ObjectInfo, - src_opts: &ObjectOptions, - dst_opts: &ObjectOptions, - ) -> Result<()>; - async fn put_object_part( - &self, - bucket: &str, - object: &str, - upload_id: &str, - part_id: usize, - data: &mut PutObjReader, - opts: &ObjectOptions, - ) -> Result; - async fn get_multipart_info( - &self, - bucket: &str, - object: &str, - upload_id: &str, - opts: &ObjectOptions, - ) -> Result; - async fn list_object_parts( - &self, - bucket: &str, - object: &str, - upload_id: &str, - part_number_marker: Option, - max_parts: usize, - opts: &ObjectOptions, - ) -> Result; - async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, opts: &ObjectOptions) -> Result<()>; - async fn complete_multipart_upload( - self: Arc, - bucket: &str, - object: &str, - upload_id: &str, - uploaded_parts: Vec, - opts: &ObjectOptions, - ) -> Result; +pub trait MultipartOperations: + rustfs_storage_api::MultipartOperations< + Error = Error, + ObjectInfo = ObjectInfo, + ObjectOptions = ObjectOptions, + PutObjectReader = PutObjReader, + CompletePart = CompletePart, + ListMultipartsInfo = ListMultipartsInfo, + MultipartUploadResult = MultipartUploadResult, + PartInfo = PartInfo, + MultipartInfo = MultipartInfo, + ListPartsInfo = ListPartsInfo, + > + Send + + Sync + + Debug +{ +} + +impl MultipartOperations for T where + T: rustfs_storage_api::MultipartOperations< + Error = Error, + ObjectInfo = ObjectInfo, + ObjectOptions = ObjectOptions, + PutObjectReader = PutObjReader, + CompletePart = CompletePart, + ListMultipartsInfo = ListMultipartsInfo, + MultipartUploadResult = MultipartUploadResult, + PartInfo = PartInfo, + MultipartInfo = MultipartInfo, + ListPartsInfo = ListPartsInfo, + > + Send + + Sync + + Debug +{ } /// Healing and repair operations. diff --git a/crates/ecstore/src/store_list_objects.rs b/crates/ecstore/src/store_list_objects.rs index 24427c955..2e80e507c 100644 --- a/crates/ecstore/src/store_list_objects.rs +++ b/crates/ecstore/src/store_list_objects.rs @@ -22,9 +22,7 @@ use crate::error::{ Error, Result, StorageError, is_all_not_found, is_all_volume_not_found, is_err_bucket_not_found, to_object_err, }; use crate::set_disk::SetDisks; -use crate::store_api::{ - ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectInfoOrErr, ObjectOperations, ObjectOptions, WalkOptions, -}; +use crate::store_api::{ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectInfoOrErr, ObjectOptions, WalkOptions}; use crate::store_utils::is_reserved_or_invalid_bucket; use crate::{store::ECStore, store_api::ListObjectsV2Info}; use futures::future::join_all; @@ -33,7 +31,7 @@ use rustfs_filemeta::{ MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams, merge_file_meta_versions, }; -use rustfs_storage_api::{VersionMarker, WalkVersionsSortOrder}; +use rustfs_storage_api::{ObjectOperations as _, VersionMarker, WalkVersionsSortOrder}; use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; diff --git a/crates/ecstore/src/tier/tier.rs b/crates/ecstore/src/tier/tier.rs index d42847960..f0d1a998e 100644 --- a/crates/ecstore/src/tier/tier.rs +++ b/crates/ecstore/src/tier/tier.rs @@ -53,6 +53,7 @@ use crate::{ store_api::{ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}, }; use rustfs_rio::HashReader; +use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join}; use s3s::S3ErrorCode; diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index 3afd5e360..60913f799 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -19,10 +19,12 @@ use rustfs_ecstore::{ disk::{DiskStore, endpoint::Endpoint}, error::StorageError, store::ECStore, - store_api::{HealOperations, ObjectIO, ObjectOperations, ObjectOptions}, + store_api::{HealOperations, ObjectOptions}, }; use rustfs_madmin::heal_commands::HealResultItem; -use rustfs_storage_api::{BucketInfo, BucketOperations, DiskSetSelector, ListOperations as _, StorageAdminApi}; +use rustfs_storage_api::{ + BucketInfo, BucketOperations, DiskSetSelector, ListOperations as _, ObjectIO as _, ObjectOperations as _, StorageAdminApi, +}; use std::sync::Arc; use tracing::{debug, error, warn}; diff --git a/crates/heal/tests/heal_integration_test.rs b/crates/heal/tests/heal_integration_test.rs index 551e64bf6..1fe5045d5 100644 --- a/crates/heal/tests/heal_integration_test.rs +++ b/crates/heal/tests/heal_integration_test.rs @@ -18,14 +18,14 @@ use rustfs_ecstore::{ disk::endpoint::Endpoint, endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, store::ECStore, - store_api::{ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}, + store_api::{ObjectOptions, PutObjReader}, }; use rustfs_heal::heal::{ manager::{HealConfig, HealManager}, storage::{ECStoreHealStorage, HealStorageAPI}, task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType}, }; -use rustfs_storage_api::BucketOperations; +use rustfs_storage_api::{BucketOperations, ObjectIO as _, ObjectOperations as _}; use serial_test::serial; use std::{ path::{Path, PathBuf}, diff --git a/crates/protocols/src/swift/object.rs b/crates/protocols/src/swift/object.rs index 1ba678fdd..df0cdd539 100644 --- a/crates/protocols/src/swift/object.rs +++ b/crates/protocols/src/swift/object.rs @@ -55,9 +55,9 @@ use super::{SwiftError, SwiftResult}; use axum::http::HeaderMap; use rustfs_credentials::Credentials; use rustfs_ecstore::resolve_object_store_handle; -use rustfs_ecstore::store_api::{ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}; +use rustfs_ecstore::store_api::{ObjectOptions, PutObjReader}; use rustfs_rio::HashReader; -use rustfs_storage_api::{BucketOperations, BucketOptions}; +use rustfs_storage_api::{BucketOperations, BucketOptions, ObjectIO as _, ObjectOperations as _}; use std::collections::HashMap; use tracing::debug; use tracing::error; diff --git a/crates/protocols/src/swift/versioning.rs b/crates/protocols/src/swift/versioning.rs index 77c56585d..7d5553fb1 100644 --- a/crates/protocols/src/swift/versioning.rs +++ b/crates/protocols/src/swift/versioning.rs @@ -57,8 +57,8 @@ use super::object::{ObjectKeyMapper, head_object}; use super::{SwiftError, SwiftResult}; use rustfs_credentials::Credentials; use rustfs_ecstore::resolve_object_store_handle; -use rustfs_ecstore::store_api::{ObjectOperations, ObjectOptions}; -use rustfs_storage_api::ListOperations as _; +use rustfs_ecstore::store_api::ObjectOptions; +use rustfs_storage_api::{ListOperations as _, ObjectOperations as _}; use std::time::{SystemTime, UNIX_EPOCH}; use tracing::{debug, error}; diff --git a/crates/s3select-api/src/object_store.rs b/crates/s3select-api/src/object_store.rs index 1308e1b1e..e2366427c 100644 --- a/crates/s3select-api/src/object_store.rs +++ b/crates/s3select-api/src/object_store.rs @@ -29,8 +29,8 @@ use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object use rustfs_ecstore::resolve_object_store_handle; use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; use rustfs_ecstore::store::ECStore; -use rustfs_ecstore::store_api::{GetObjectReader, ObjectIO, ObjectOperations, ObjectOptions}; -use rustfs_storage_api::HTTPRangeSpec; +use rustfs_ecstore::store_api::{GetObjectReader, ObjectOptions}; +use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _}; use s3s::S3Result; use s3s::dto::SelectObjectContentInput; use s3s::header::{ diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 4a6e36bc0..266262340 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -1069,7 +1069,7 @@ pub async fn store_data_usage_in_backend( #[cfg(test)] mod tests { use super::*; - use rustfs_ecstore::store_api::{GetObjectReader, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader}; + use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader}; use serial_test::serial; use std::collections::HashMap; use std::io::Cursor; @@ -1119,7 +1119,15 @@ mod tests { } #[async_trait::async_trait] - impl ObjectIO for MemoryConfigStore { + impl rustfs_storage_api::ObjectIO for MemoryConfigStore { + type Error = rustfs_ecstore::error::Error; + type RangeSpec = rustfs_storage_api::HTTPRangeSpec; + type HeaderMap = http::HeaderMap; + type ObjectOptions = ObjectOptions; + type ObjectInfo = ObjectInfo; + type GetObjectReader = GetObjectReader; + type PutObjectReader = PutObjReader; + async fn get_object_reader( &self, bucket: &str, diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index 3c23781b0..f08a4ad35 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -28,7 +28,7 @@ use rustfs_ecstore::{ global::GLOBAL_TierConfigMgr, pools::path2_bucket_object_with_base_path, store::ECStore, - store_api::{MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}, + store_api::{ObjectOptions, PutObjReader}, tier::{ tier_config::{TierConfig, TierMinIO, TierType}, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, @@ -38,8 +38,9 @@ use rustfs_filemeta::FileMeta; use rustfs_scanner::scanner::init_data_scanner; use rustfs_scanner::scanner_folder::ScannerItem; use rustfs_scanner::scanner_io::ScannerIODisk; -use rustfs_storage_api::ListOperations as _; -use rustfs_storage_api::{BucketOperations, MakeBucketOptions}; +use rustfs_storage_api::{ + BucketOperations, ListOperations as _, MakeBucketOptions, MultipartOperations as _, ObjectIO as _, ObjectOperations as _, +}; use rustfs_utils::path::path_join_buf; use s3s::dto::RestoreRequest; use serial_test::serial; diff --git a/crates/storage-api/src/lib.rs b/crates/storage-api/src/lib.rs index 0f732796d..1a54d71ab 100644 --- a/crates/storage-api/src/lib.rs +++ b/crates/storage-api/src/lib.rs @@ -26,5 +26,6 @@ pub use error::{StorageErrorCode, StorageResult}; pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo}; pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions}; pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr}; +pub use object::{MultipartOperations, ObjectIO, ObjectOperations}; pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState}; pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder}; diff --git a/crates/storage-api/src/object.rs b/crates/storage-api/src/object.rs index 58a73247e..acec1802e 100644 --- a/crates/storage-api/src/object.rs +++ b/crates/storage-api/src/object.rs @@ -180,6 +180,106 @@ impl Default for WalkOptions { } } +#[async_trait::async_trait] +pub trait ObjectIO: Send + Sync + fmt::Debug + 'static { + type Error: std::error::Error + Send + Sync + 'static; + type RangeSpec: Send + 'static; + type HeaderMap: Send + 'static; + type ObjectOptions: Send + Sync + 'static; + type ObjectInfo: Send + 'static; + type GetObjectReader: Send + 'static; + type PutObjectReader: Send + 'static; + + async fn get_object_reader( + &self, + bucket: &str, + object: &str, + range: Option, + h: Self::HeaderMap, + opts: &Self::ObjectOptions, + ) -> Result; + + async fn put_object( + &self, + bucket: &str, + object: &str, + data: &mut Self::PutObjectReader, + opts: &Self::ObjectOptions, + ) -> Result; +} + +#[async_trait::async_trait] +#[allow(clippy::too_many_arguments)] +pub trait ObjectOperations: Send + Sync + fmt::Debug { + type Error: std::error::Error + Send + Sync + 'static; + type ObjectInfo: Send + 'static; + type ObjectOptions: Send + Sync + 'static; + type FileInfo: Send + Sync + 'static; + type ObjectToDelete: Send + 'static; + type DeletedObject: Send + 'static; + + async fn get_object_info( + &self, + bucket: &str, + object: &str, + opts: &Self::ObjectOptions, + ) -> Result; + async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &Self::ObjectOptions) -> Result<(), Self::Error>; + async fn copy_object( + &self, + src_bucket: &str, + src_object: &str, + dst_bucket: &str, + dst_object: &str, + src_info: &mut Self::ObjectInfo, + src_opts: &Self::ObjectOptions, + dst_opts: &Self::ObjectOptions, + ) -> Result; + async fn delete_object_version( + &self, + bucket: &str, + object: &str, + fi: &Self::FileInfo, + force_del_marker: bool, + ) -> Result<(), Self::Error>; + async fn delete_object(&self, bucket: &str, object: &str, opts: Self::ObjectOptions) + -> Result; + async fn delete_objects( + &self, + bucket: &str, + objects: Vec, + opts: Self::ObjectOptions, + ) -> (Vec, Vec>); + async fn put_object_metadata( + &self, + bucket: &str, + object: &str, + opts: &Self::ObjectOptions, + ) -> Result; + async fn get_object_tags(&self, bucket: &str, object: &str, opts: &Self::ObjectOptions) -> Result; + async fn put_object_tags( + &self, + bucket: &str, + object: &str, + tags: &str, + opts: &Self::ObjectOptions, + ) -> Result; + async fn delete_object_tags( + &self, + bucket: &str, + object: &str, + opts: &Self::ObjectOptions, + ) -> Result; + async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<(), Self::Error>; + async fn transition_object(&self, bucket: &str, object: &str, opts: &Self::ObjectOptions) -> Result<(), Self::Error>; + async fn restore_transitioned_object( + self: Arc, + bucket: &str, + object: &str, + opts: &Self::ObjectOptions, + ) -> Result<(), Self::Error>; +} + #[async_trait::async_trait] #[allow(clippy::too_many_arguments)] pub trait ListOperations: Send + Sync + fmt::Debug { @@ -223,6 +323,91 @@ pub trait ListOperations: Send + Sync + fmt::Debug { ) -> Result<(), Self::Error>; } +#[async_trait::async_trait] +#[allow(clippy::too_many_arguments)] +pub trait MultipartOperations: Send + Sync + fmt::Debug { + type Error: std::error::Error + Send + Sync + 'static; + type ObjectInfo: Send + 'static; + type ObjectOptions: Send + Sync + 'static; + type PutObjectReader: Send + 'static; + type CompletePart: Send + 'static; + type ListMultipartsInfo: Send + 'static; + type MultipartUploadResult: Send + 'static; + type PartInfo: Send + 'static; + type MultipartInfo: Send + 'static; + type ListPartsInfo: Send + 'static; + + async fn list_multipart_uploads( + &self, + bucket: &str, + prefix: &str, + key_marker: Option, + upload_id_marker: Option, + delimiter: Option, + max_uploads: usize, + ) -> Result; + async fn new_multipart_upload( + &self, + bucket: &str, + object: &str, + opts: &Self::ObjectOptions, + ) -> Result; + async fn copy_object_part( + &self, + src_bucket: &str, + src_object: &str, + dst_bucket: &str, + dst_object: &str, + upload_id: &str, + part_id: usize, + start_offset: i64, + length: i64, + src_info: &Self::ObjectInfo, + src_opts: &Self::ObjectOptions, + dst_opts: &Self::ObjectOptions, + ) -> Result<(), Self::Error>; + async fn put_object_part( + &self, + bucket: &str, + object: &str, + upload_id: &str, + part_id: usize, + data: &mut Self::PutObjectReader, + opts: &Self::ObjectOptions, + ) -> Result; + async fn get_multipart_info( + &self, + bucket: &str, + object: &str, + upload_id: &str, + opts: &Self::ObjectOptions, + ) -> Result; + async fn list_object_parts( + &self, + bucket: &str, + object: &str, + upload_id: &str, + part_number_marker: Option, + max_parts: usize, + opts: &Self::ObjectOptions, + ) -> Result; + async fn abort_multipart_upload( + &self, + bucket: &str, + object: &str, + upload_id: &str, + opts: &Self::ObjectOptions, + ) -> Result<(), Self::Error>; + async fn complete_multipart_upload( + self: Arc, + bucket: &str, + object: &str, + upload_id: &str, + uploaded_parts: Vec, + opts: &Self::ObjectOptions, + ) -> Result; +} + #[derive(Debug, Default)] pub struct ListObjectsInfo { pub is_truncated: bool, diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index c9b7e15f7..a4cc6b139 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,18 +5,19 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-list-operations-contracts` -- Baseline: `main` at `36f7ad6936d03b9593880b5c4958f47b2a039fc9` - after the walk options contract merge. +- Branch: `overtrue/arch-object-operation-contracts` +- Baseline: `main` at `3fb4cb3d65a2e037fc2e5ede32bf81c1f15c9fb7` + after the list operations contract merge. - PR type for this branch: `api-extraction` - Runtime behavior changes: no external behavior change expected. -- Rust code changes: move `ListOperations` into `rustfs-storage-api` as a - generic operation contract with associated ECStore-bound types, then keep - ECStore's existing public name as a fixed associated-type compatibility - subtrait. -- CI/script changes: extend migration guards for the `ListOperations` public - re-export and ECStore local-definition regressions. -- Docs changes: record the list operations contract extraction slice. +- Rust code changes: move `ObjectIO`, `ObjectOperations`, and + `MultipartOperations` into `rustfs-storage-api` as generic operation + contracts with associated ECStore-bound types, then keep ECStore's existing + public names as fixed associated-type compatibility subtraits. +- CI/script changes: extend migration guards for the object/multipart operation + public re-exports and ECStore local-method regressions. +- Docs changes: record the object and multipart operation contract extraction + slice. ## Phase 0 Tasks @@ -684,6 +685,31 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block checks, migration/layer guards, formatting, diff hygiene, Rust risk scan, full pre-commit, and required three-expert review passed. +- [x] `API-022` Move object and multipart operation contracts. + - Completed slice: move `ObjectIO`, `ObjectOperations`, and + `MultipartOperations` from ECStore `store_api/traits.rs` into + `rustfs-storage-api` as generic public operation contracts over ECStore + reader, option, metadata, multipart DTO, file-info, delete, header, range, + and error associated types; keep ECStore's old public trait names as fixed + associated-type compatibility subtraits. + - Acceptance: `rustfs-storage-api` exports the object and multipart + operation contracts, ECStore no longer defines local object/multipart method + signatures, existing ECStore generic bounds keep the old import path, and + migration guards reject dropping the public storage-api re-export or + reintroducing local ECStore object/multipart method definitions. + - Must preserve: object reader/writer behavior, object metadata/tag/delete + behavior, multipart create/copy/part/list/complete/abort behavior, ECStore + public compatibility bounds, and all ECStore object/multipart runtime + behavior. + - Risk defense: only the trait contracts cross into `rustfs-storage-api`; + ECStore keeps the concrete associated type bindings, readers, + `ObjectInfo`, `ObjectOptions`, `PutObjReader`, filemeta adaptation, storage + errors, lifecycle/replication/rio/compression/encryption coupling, and + implementation bodies. + - Verification: focused storage-api tests, ECStore/RustFS/downstream compile + checks, migration/layer guards, formatting, diff hygiene, Rust risk scan, + full pre-commit, and required three-expert review passed. + ## Phase 8 Background Controller Tasks - [x] `BGC-001` Inventory background services. @@ -960,8 +986,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | Generic `ListOperations` now lives in `rustfs-storage-api`; ECStore still owns the concrete associated type bindings and implementation behavior. | -| Migration preservation | passed | List v2, list-object-versions, walk channel/cancellation shape, and existing ECStore generic bound import path are preserved through a compatibility subtrait. | +| Quality/architecture | passed | Generic object I/O, object operation, and multipart operation traits now live in `rustfs-storage-api`; ECStore still owns concrete type bindings and implementations. | +| Migration preservation | passed | Object read/write, metadata/tag/delete, multipart, and existing ECStore generic bound import paths are preserved through compatibility subtraits. | | Testing/verification | passed | Focused storage-api tests, downstream compile checks, migration/layer guards, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` passed. | ## Verification Notes @@ -969,7 +995,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block Passed before push: - `cargo test -p rustfs-storage-api`: passed. -- `cargo check --tests -p rustfs-storage-api -p rustfs-ecstore -p rustfs -p rustfs-iam -p rustfs-scanner -p rustfs-protocols`: passed. +- `cargo check --tests -p rustfs-storage-api -p rustfs-ecstore -p rustfs -p rustfs-scanner -p rustfs-protocols`: passed. +- `cargo check --tests -p rustfs-heal -p rustfs-protocols`: passed. - `./scripts/check_architecture_migration_rules.sh`: passed. - `./scripts/check_layer_dependencies.sh`: passed. - `cargo fmt --all --check`: passed. @@ -982,22 +1009,23 @@ Passed before push: Notes: -- This slice follows the walk options contract branch and keeps the old +- This slice follows the list operations contract branch and keeps the old aggregate facade, bucket DTO, multipart DTO, bucket operation contract, object helper, range helper, list helper, object precondition, list response, and - walk options contract guards active. -- The shared list operations contract is now owned by `rustfs-storage-api`; - ECStore keeps the concrete associated type bindings, response aliases, - `WalkOptions` alias, `ObjectInfo`, storage `Error`, `ObjectOptions`, object - metadata adaptation, storage error mapping, readers, lifecycle/replication, - rio, filemeta, and implementation behavior. + walk/list operation contract guards active. +- The shared object and multipart operation contracts are now owned by + `rustfs-storage-api`; ECStore keeps the concrete associated type bindings, + readers, `ObjectInfo`, `ObjectOptions`, `PutObjReader`, filemeta adaptation, + storage error mapping, lifecycle/replication, rio, compression/encryption, + and implementation behavior. - The slice does not alter object, list, walk, multipart, bucket, delete, - namespace-lock, or reader runtime behavior. + namespace-lock, reader, or tag/metadata runtime behavior. ## Handoff Notes -- List operations contract cleanup is stacked on the walk options contract +- Object and multipart operation contract cleanup is stacked on the list + operations contract branch. - After this lands, remaining storage work can continue by extracting larger - low-coupling DTO/consumer slices or by narrowing remaining operation-group - consumers. + low-coupling object metadata/option/reader slices or by narrowing remaining + operation-group consumers. diff --git a/rustfs/src/admin/handlers/object_zip_download.rs b/rustfs/src/admin/handlers/object_zip_download.rs index 96d84267f..01f6fca86 100644 --- a/rustfs/src/admin/handlers/object_zip_download.rs +++ b/rustfs/src/admin/handlers/object_zip_download.rs @@ -31,12 +31,9 @@ use matchit::Params; use rand::RngExt; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_credentials::get_global_action_cred; -use rustfs_ecstore::{ - global::get_global_region, - store_api::{ObjectIO, ObjectOperations, ObjectOptions}, -}; +use rustfs_ecstore::{global::get_global_region, store_api::ObjectOptions}; use rustfs_policy::policy::action::{Action, S3Action}; -use rustfs_storage_api::{BucketOperations, ListOperations as _, bucket::BucketOptions}; +use rustfs_storage_api::{BucketOperations, ListOperations as _, ObjectIO as _, ObjectOperations as _, bucket::BucketOptions}; use rustfs_trusted_proxies::{ClientInfo, ValidationMode}; use rustfs_utils::{base64_decode_url_safe_no_pad, base64_encode_url_safe_no_pad}; use s3s::{Body, S3Request, S3Response, S3Result, dto::StreamingBlob, header::CONTENT_TYPE, s3_error}; diff --git a/rustfs/src/app/capacity_dirty_scope_test.rs b/rustfs/src/app/capacity_dirty_scope_test.rs index 367c1c98f..1ae627255 100644 --- a/rustfs/src/app/capacity_dirty_scope_test.rs +++ b/rustfs/src/app/capacity_dirty_scope_test.rs @@ -18,10 +18,10 @@ use rustfs_ecstore::{ disk::endpoint::Endpoint, endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, store::ECStore, - store_api::{HealOperations, ObjectIO, ObjectOptions, PutObjReader}, + store_api::{HealOperations, ObjectOptions, PutObjReader}, }; use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager}; -use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions}; +use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions, ObjectIO as _}; use serial_test::serial; use std::{ collections::HashSet, diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index b8ed15625..d1546e660 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -29,15 +29,17 @@ use rustfs_ecstore::{ endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, global::GLOBAL_TierConfigMgr, store::ECStore, - store_api::{MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}, + store_api::{ObjectOptions, PutObjReader}, tier::{ tier_config::{TierConfig, TierType}, warm_backend::{WarmBackend, WarmBackendGetOpts}, }, }; use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager}; -use rustfs_storage_api::ListOperations as _; -use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions}; +use rustfs_storage_api::{ + BucketOperations, BucketOptions, ListOperations as _, MakeBucketOptions, MultipartOperations as _, ObjectIO as _, + ObjectOperations as _, +}; use rustfs_utils::http::{SUFFIX_FORCE_DELETE, insert_header}; use s3s::{S3Request, dto::*}; use serial_test::serial; diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 4d0f2fcc5..4bb5fb421 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -53,13 +53,14 @@ use rustfs_ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_r use rustfs_ecstore::rio::{HashReader, WritePlan}; use rustfs_ecstore::set_disk::is_valid_storage_class; use rustfs_ecstore::store::ECStore; -use rustfs_ecstore::store_api::{MultipartOperations, ObjectOperations}; -use rustfs_ecstore::store_api::{ObjectIO, ObjectOptions, PutObjReader}; +use rustfs_ecstore::store_api::{ObjectOptions, PutObjReader}; use rustfs_filemeta::{ReplicationStatusType, ReplicationType}; use rustfs_s3_ops::S3Operation; #[cfg(test)] use rustfs_storage_api::HTTPPreconditions; -use rustfs_storage_api::{CompletePart, HTTPRangeSpec, MultipartUploadResult}; +use rustfs_storage_api::{ + CompletePart, HTTPRangeSpec, MultipartOperations as _, MultipartUploadResult, ObjectIO as _, ObjectOperations as _, +}; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; use rustfs_utils::http::{ diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 0d31748f1..b55435a19 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -78,9 +78,7 @@ use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object use rustfs_ecstore::rio::{DynReader, HashReader, WritePlan, wrap_reader}; use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class}; use rustfs_ecstore::store::ECStore; -use rustfs_ecstore::store_api::{ - NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader, -}; +use rustfs_ecstore::store_api::{NamespaceLocking, ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader}; use rustfs_filemeta::{ REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType, ReplicationType, RestoreStatusOps, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map, @@ -95,7 +93,7 @@ use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_fo use rustfs_s3select_api::object_store::bytes_stream; #[cfg(test)] use rustfs_storage_api::HTTPPreconditions; -use rustfs_storage_api::HTTPRangeSpec; +use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _}; use rustfs_targets::{ EventName, extract_params_header, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent, }; diff --git a/rustfs/src/app/select_object.rs b/rustfs/src/app/select_object.rs index 4f46f1be4..9de3ea71b 100644 --- a/rustfs/src/app/select_object.rs +++ b/rustfs/src/app/select_object.rs @@ -10,13 +10,13 @@ use datafusion::arrow::{ }; use futures::StreamExt; use http::{StatusCode, header::RANGE}; -use rustfs_ecstore::store_api::ObjectOperations; use rustfs_s3select_api::{ QueryError, object_store::{INVALID_SCAN_RANGE_MESSAGE, validate_scan_range_bounds}, query::{Context, Query}, }; use rustfs_s3select_query::get_global_db; +use rustfs_storage_api::ObjectOperations as _; use s3s::dto::*; use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use std::sync::Arc; diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 8af171766..f511cc1fc 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -40,11 +40,11 @@ use rustfs_ecstore::{ versioning_sys::BucketVersioningSys, }, error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found}, - store_api::{ObjectOperations, ObjectOptions}, + store_api::ObjectOptions, }; use rustfs_io_metrics::record_s3_op; use rustfs_s3_ops::S3Operation; -use rustfs_storage_api::{BucketOperations, BucketOptions, ObjectLockRetentionOptions}; +use rustfs_storage_api::{BucketOperations, BucketOptions, ObjectLockRetentionOptions, ObjectOperations as _}; use rustfs_targets::EventName; use rustfs_utils::http::headers::{ AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 277b7125a..9b1fb93ae 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -57,6 +57,7 @@ STORE_API_OBJECT_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_object_helper_reexp STORE_API_RANGE_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_range_helper_reexports.txt" STORE_API_LIST_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_list_helper_reexports.txt" STORE_API_LIST_RESPONSE_REEXPORTS_FILE="${TMP_DIR}/store_api_list_response_reexports.txt" +STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE="${TMP_DIR}/store_api_object_operation_local_method_hits.txt" awk ' /^## PR Types$/ { @@ -201,6 +202,10 @@ require_source_line \ "crates/storage-api/src/lib.rs" \ "pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};" \ "storage-api public list response contract re-export" +require_source_line \ + "crates/storage-api/src/lib.rs" \ + "pub use object::{MultipartOperations, ObjectIO, ObjectOperations};" \ + "storage-api public object operation contract re-export" require_source_line \ "crates/storage-api/src/lib.rs" \ "pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};" \ @@ -293,6 +298,28 @@ if [[ -s "$STORE_API_LIST_RESPONSE_REEXPORTS_FILE" ]]; then report_failure "old ecstore store_api list response path reintroduced: $(paste -sd '; ' "$STORE_API_LIST_RESPONSE_REEXPORTS_FILE")" fi +( + cd "$ROOT_DIR" + rg -n --no-heading 'async fn (get_object_reader|put_object|get_object_info|verify_object_integrity|copy_object|delete_object_version|delete_object|delete_objects|put_object_metadata|get_object_tags|put_object_tags|delete_object_tags|add_partial|transition_object|restore_transitioned_object|list_multipart_uploads|new_multipart_upload|copy_object_part|put_object_part|get_multipart_info|list_object_parts|abort_multipart_upload|complete_multipart_upload)\b' \ + crates/ecstore/src/store_api/traits.rs || true +) >"$STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE" + +if [[ -s "$STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE" ]]; then + report_failure "old ecstore object/multipart operation method signatures reintroduced: $(paste -sd '; ' "$STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE")" +fi + +require_source_contains \ + "crates/ecstore/src/store_api/traits.rs" \ + "rustfs_storage_api::ObjectIO<" \ + "ECStore ObjectIO compatibility binding" +require_source_contains \ + "crates/ecstore/src/store_api/traits.rs" \ + "rustfs_storage_api::ObjectOperations<" \ + "ECStore ObjectOperations compatibility binding" +require_source_contains \ + "crates/ecstore/src/store_api/traits.rs" \ + "rustfs_storage_api::MultipartOperations<" \ + "ECStore MultipartOperations compatibility binding" require_source_contains \ "crates/ecstore/src/store_api/traits.rs" \ "pub trait NamespaceLocking: Send + Sync + Debug + 'static" \