refactor: move object operation contracts (#3559)

This commit is contained in:
安正超
2026-06-18 09:48:13 +08:00
committed by GitHub
parent 3fb4cb3d65
commit e5cad7ed20
34 changed files with 529 additions and 200 deletions
@@ -37,7 +37,7 @@ use crate::global::GLOBAL_LocalNodeName;
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks};
use crate::store::ECStore;
use crate::store_api::{GetObjectReader, MultipartOperations, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete};
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, ObjectToDelete};
use crate::tier::warm_backend::WarmBackendGetOpts;
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
use futures::Future;
@@ -58,7 +58,7 @@ use rustfs_filemeta::{
VersionPurgeStatusType, get_file_info, is_restored_object_on_disk,
};
use rustfs_s3_types::EventName;
use rustfs_storage_api::{HTTPRangeSpec, ListOperations as _};
use rustfs_storage_api::{HTTPRangeSpec, ListOperations as _, MultipartOperations as _, ObjectOperations as _};
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest,
@@ -2870,12 +2870,12 @@ mod tests {
use crate::error::is_err_invalid_upload_id;
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
use crate::store::ECStore;
use crate::store_api::{MultipartOperations, ObjectInfo, ObjectOptions, PutObjReader};
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader};
use futures::FutureExt;
use rustfs_common::metrics::{IlmAction, global_metrics};
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
use rustfs_filemeta::{ReplicateDecision, VersionPurgeStatusType};
use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions};
use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations as _};
use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, Timestamp};
use serial_test::serial;
use sha2::{Digest, Sha256};
@@ -25,9 +25,10 @@ use crate::bucket::replication::{DeletedObjectReplicationInfo, check_replicate_d
use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::store::ECStore;
use crate::store_api::{ObjectOperations, ObjectOptions, ObjectToDelete};
use crate::store_api::{ObjectOptions, ObjectToDelete};
use rustfs_filemeta::{REPLICATE_INCOMING_DELETE, ReplicationState, version_purge_statuses_map};
use rustfs_lock::MAX_DELETE_LIST;
use rustfs_storage_api::ObjectOperations as _;
fn lifecycle_version_delete_replication_state(
replicate_decision_str: String,
+10 -2
View File
@@ -1217,7 +1217,7 @@ mod tests {
use crate::error::{Error, Result};
use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
use crate::set_disk::SetDisks;
use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader};
use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectInfo, ObjectOptions, PutObjReader};
use http::HeaderMap;
use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{
@@ -1408,7 +1408,15 @@ mod tests {
}
#[async_trait::async_trait]
impl ObjectIO for LockingConfigStorage {
impl rustfs_storage_api::ObjectIO for LockingConfigStorage {
type Error = Error;
type RangeSpec = HTTPRangeSpec;
type HeaderMap = HeaderMap;
type ObjectOptions = ObjectOptions;
type ObjectInfo = ObjectInfo;
type GetObjectReader = GetObjectReader;
type PutObjectReader = PutObjReader;
async fn get_object_reader(
&self,
bucket: &str,
+2 -4
View File
@@ -14,12 +14,10 @@
use crate::error::{Error, Result, is_err_data_movement_overwrite, is_err_object_not_found, is_err_version_not_found};
use crate::store::ECStore;
use crate::store_api::{
GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, PutObjReader,
};
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use bytes::Bytes;
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
use rustfs_storage_api::CompletePart;
use rustfs_storage_api::{CompletePart, MultipartOperations as _, ObjectIO as _, ObjectOperations as _};
use rustfs_utils::path::encode_dir_object;
use std::io::Cursor;
use std::pin::Pin;
+2 -2
View File
@@ -30,7 +30,7 @@ use rustfs_data_usage::{
BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary,
};
use rustfs_io_metrics::record_system_path_failure;
use rustfs_storage_api::ListOperations as _;
use rustfs_storage_api::{ListOperations as _, ObjectIO as _};
use rustfs_utils::path::SLASH_SEPARATOR;
use std::{
collections::{HashMap, HashSet, hash_map::Entry},
@@ -715,7 +715,7 @@ pub fn cache_to_data_usage_info(cache: &DataUsageCache, path: &str, buckets: &[r
// Helper functions for DataUsageCache operations
pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str) -> crate::error::Result<DataUsageCache> {
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::store_api::{ObjectIO, ObjectOptions};
use crate::store_api::ObjectOptions;
use http::HeaderMap;
use rand::RngExt;
use std::path::Path;
+4 -2
View File
@@ -38,7 +38,7 @@ use crate::error::{
use crate::notification_sys::get_global_notification_sys;
use crate::resolve_object_store_handle;
use crate::set_disk::SetDisks;
use crate::store_api::{GetObjectReader, HealOperations, ObjectIO, ObjectOperations, ObjectOptions};
use crate::store_api::{GetObjectReader, HealOperations, ObjectOptions};
use crate::{global::GLOBAL_LifecycleSys, sets::Sets, store::ECStore};
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered};
@@ -50,7 +50,9 @@ use rustfs_common::defer;
use rustfs_common::heal_channel::HealOpts;
use rustfs_concurrency::workers::Workers;
use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions, StorageAdminApi};
use rustfs_storage_api::{
BucketOperations, BucketOptions, MakeBucketOptions, ObjectIO as _, ObjectOperations as _, StorageAdminApi,
};
use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path};
use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration};
use serde::{Deserialize, Serialize};
+2 -2
View File
@@ -23,11 +23,11 @@ use crate::global::get_global_endpoints;
use crate::pools::ListCallback;
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
use crate::store::ECStore;
use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions};
use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOptions};
use http::HeaderMap;
use rand::RngExt as _;
use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_storage_api::{HTTPRangeSpec, StorageAdminApi};
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _, StorageAdminApi};
use rustfs_utils::path::encode_dir_object;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
+33 -5
View File
@@ -90,6 +90,7 @@ use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo,
MakeBucketOptions, MultipartInfo, MultipartUploadResult, PartInfo,
};
use rustfs_storage_api::{MultipartOperations as _, ObjectIO as _, ObjectOperations as _};
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
use rustfs_utils::http::headers::{
@@ -828,7 +829,15 @@ fn classify_small_write_path(is_inline_buffer: bool, object_size: i64, block_siz
}
#[async_trait::async_trait]
impl ObjectIO for SetDisks {
impl rustfs_storage_api::ObjectIO for SetDisks {
type Error = Error;
type RangeSpec = HTTPRangeSpec;
type HeaderMap = HeaderMap;
type ObjectOptions = ObjectOptions;
type ObjectInfo = ObjectInfo;
type GetObjectReader = GetObjectReader;
type PutObjectReader = PutObjReader;
#[tracing::instrument(level = "debug", skip(self))]
async fn get_object_reader(
&self,
@@ -1791,7 +1800,14 @@ fn check_object_lock_retention_update(bucket: &str, object: &str, obj_info: &Obj
}
#[async_trait::async_trait]
impl ObjectOperations for SetDisks {
impl rustfs_storage_api::ObjectOperations for SetDisks {
type Error = Error;
type ObjectInfo = ObjectInfo;
type ObjectOptions = ObjectOptions;
type FileInfo = FileInfo;
type ObjectToDelete = ObjectToDelete;
type DeletedObject = DeletedObject;
#[tracing::instrument(skip(self))]
async fn copy_object(
&self,
@@ -2866,7 +2882,8 @@ impl ObjectOperations for SetDisks {
#[tracing::instrument(skip(self))]
async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
let get_object_reader = <Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
let get_object_reader =
<Self as rustfs_storage_api::ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
// Stream to sink to avoid loading entire object into memory during verification
let mut reader = get_object_reader.stream;
tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
@@ -3035,7 +3052,18 @@ impl rustfs_storage_api::ListOperations for SetDisks {
}
#[async_trait::async_trait]
impl MultipartOperations for SetDisks {
impl rustfs_storage_api::MultipartOperations for SetDisks {
type Error = Error;
type ObjectInfo = ObjectInfo;
type ObjectOptions = ObjectOptions;
type PutObjectReader = PutObjReader;
type CompletePart = CompletePart;
type ListMultipartsInfo = ListMultipartsInfo;
type MultipartUploadResult = MultipartUploadResult;
type PartInfo = PartInfo;
type MultipartInfo = MultipartInfo;
type ListPartsInfo = ListPartsInfo;
#[tracing::instrument(skip(self))]
async fn copy_object_part(
&self,
@@ -4991,7 +5019,7 @@ mod tests {
use rustfs_filemeta::ReplicationState;
use rustfs_lock::client::local::LocalClient;
use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};
use rustfs_storage_api::CompletePart;
use rustfs_storage_api::{CompletePart, ObjectOperations as _};
use serial_test::serial;
use std::collections::HashMap;
use tempfile::TempDir;
+32 -5
View File
@@ -28,8 +28,8 @@ use crate::{
global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure},
set_disk::SetDisks,
store_api::{
DeletedObject, GetObjectReader, HealOperations, ListObjectVersionsInfo, ListObjectsV2Info, MultipartOperations,
NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader,
DeletedObject, GetObjectReader, HealOperations, ListObjectVersionsInfo, ListObjectsV2Info, NamespaceLocking, ObjectInfo,
ObjectOptions, ObjectToDelete, PutObjReader,
},
store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
};
@@ -53,6 +53,7 @@ use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo, MakeBucketOptions,
MultipartInfo, MultipartUploadResult, PartInfo,
};
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
use std::{collections::HashMap, sync::Arc};
use tokio::sync::RwLock;
@@ -402,7 +403,15 @@ fn apply_delete_objects_results(
}
#[async_trait::async_trait]
impl ObjectIO for Sets {
impl rustfs_storage_api::ObjectIO for Sets {
type Error = Error;
type RangeSpec = HTTPRangeSpec;
type HeaderMap = HeaderMap;
type ObjectOptions = ObjectOptions;
type ObjectInfo = ObjectInfo;
type GetObjectReader = GetObjectReader;
type PutObjectReader = PutObjReader;
#[tracing::instrument(level = "debug", skip(self, object, h, opts))]
async fn get_object_reader(
&self,
@@ -447,7 +456,14 @@ impl BucketOperations for Sets {
}
#[async_trait::async_trait]
impl ObjectOperations for Sets {
impl rustfs_storage_api::ObjectOperations for Sets {
type Error = Error;
type ObjectInfo = ObjectInfo;
type ObjectOptions = ObjectOptions;
type FileInfo = FileInfo;
type ObjectToDelete = ObjectToDelete;
type DeletedObject = DeletedObject;
async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.get_disks_by_key(object).get_object_info(bucket, object, opts).await
}
@@ -696,7 +712,18 @@ impl rustfs_storage_api::ListOperations for Sets {
}
#[async_trait::async_trait]
impl MultipartOperations for Sets {
impl rustfs_storage_api::MultipartOperations for Sets {
type Error = Error;
type ObjectInfo = ObjectInfo;
type ObjectOptions = ObjectOptions;
type PutObjectReader = PutObjReader;
type CompletePart = CompletePart;
type ListMultipartsInfo = ListMultipartsInfo;
type MultipartUploadResult = MultipartUploadResult;
type PartInfo = PartInfo;
type MultipartInfo = MultipartInfo;
type ListPartsInfo = ListPartsInfo;
#[tracing::instrument(skip(self))]
async fn list_multipart_uploads(
&self,
+32 -6
View File
@@ -53,7 +53,7 @@ use crate::notification_sys::get_global_notification_sys;
use crate::pools::PoolMeta;
use crate::rebalance::RebalanceMeta;
use crate::rpc::RemoteClient;
use crate::store_api::{ListObjectVersionsInfo, ObjectIO, ObjectInfoOrErr, WalkOptions};
use crate::store_api::{ListObjectVersionsInfo, ObjectInfoOrErr, WalkOptions};
use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
use crate::tier::tier::TierConfigMgr;
use crate::{
@@ -63,8 +63,8 @@ use crate::{
rpc::S3PeerSys,
sets::Sets,
store_api::{
DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, MultipartOperations, NamespaceLocking, ObjectInfo,
ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader,
DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, NamespaceLocking, ObjectInfo, ObjectOptions,
ObjectToDelete, PutObjReader,
},
store_init,
};
@@ -367,7 +367,15 @@ impl Clone for PoolObjInfo {
// }
#[async_trait::async_trait]
impl ObjectIO for ECStore {
impl rustfs_storage_api::ObjectIO for ECStore {
type Error = Error;
type RangeSpec = HTTPRangeSpec;
type HeaderMap = HeaderMap;
type ObjectOptions = ObjectOptions;
type ObjectInfo = ObjectInfo;
type GetObjectReader = GetObjectReader;
type PutObjectReader = PutObjReader;
#[instrument(level = "debug", skip(self))]
async fn get_object_reader(
&self,
@@ -420,7 +428,14 @@ impl BucketOperations for ECStore {
}
#[async_trait::async_trait]
impl ObjectOperations for ECStore {
impl rustfs_storage_api::ObjectOperations for ECStore {
type Error = Error;
type ObjectInfo = ObjectInfo;
type ObjectOptions = ObjectOptions;
type FileInfo = FileInfo;
type ObjectToDelete = ObjectToDelete;
type DeletedObject = DeletedObject;
async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
self.handle_get_object_info(bucket, object, opts).await
}
@@ -569,7 +584,18 @@ impl rustfs_storage_api::ListOperations for ECStore {
}
#[async_trait::async_trait]
impl MultipartOperations for ECStore {
impl rustfs_storage_api::MultipartOperations for ECStore {
type Error = Error;
type ObjectInfo = ObjectInfo;
type ObjectOptions = ObjectOptions;
type PutObjectReader = PutObjReader;
type CompletePart = CompletePart;
type ListMultipartsInfo = ListMultipartsInfo;
type MultipartUploadResult = MultipartUploadResult;
type PartInfo = PartInfo;
type MultipartInfo = MultipartInfo;
type ListPartsInfo = ListPartsInfo;
#[instrument(skip(self))]
async fn list_multipart_uploads(
&self,
+1
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use super::*;
use rustfs_storage_api::MultipartOperations as _;
impl ECStore {
#[instrument(skip(self))]
+3 -1
View File
@@ -21,6 +21,7 @@ use rustfs_io_metrics::{
record_object_lock_diag_acquire_duration, record_object_lock_diag_hold_duration, record_object_lock_diag_slow_acquire,
record_object_lock_diag_slow_hold,
};
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
use std::{
fmt,
pin::Pin,
@@ -1254,7 +1255,8 @@ impl ECStore {
}
pub(super) async fn handle_verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
let get_object_reader = <Self as ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
let get_object_reader =
<Self as rustfs_storage_api::ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
// Stream to sink to avoid loading entire object into memory during verification
let mut reader = get_object_reader.stream;
tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
+1 -1
View File
@@ -14,7 +14,7 @@
use super::*;
use crate::config::get_global_storage_class;
use rustfs_storage_api::StorageAdminApi;
use rustfs_storage_api::{ObjectOperations as _, StorageAdminApi};
struct LatestObjectInfoCandidate {
info: Option<ObjectInfo>,
+89 -102
View File
@@ -1,51 +1,65 @@
use super::*;
use rustfs_storage_api::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
#[async_trait::async_trait]
pub trait ObjectIO: Send + Sync + Debug + 'static {
async fn get_object_reader(
&self,
bucket: &str,
object: &str,
range: Option<HTTPRangeSpec>,
h: HeaderMap,
opts: &ObjectOptions,
) -> Result<GetObjectReader>;
pub trait ObjectIO:
rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo>;
impl<T> ObjectIO for T where
T: rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
/// Object-level storage operations (beyond basic I/O in ObjectIO).
#[async_trait::async_trait]
#[allow(clippy::too_many_arguments)]
pub trait ObjectOperations: Send + Sync + Debug {
async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>;
async fn copy_object(
&self,
src_bucket: &str,
src_object: &str,
dst_bucket: &str,
dst_object: &str,
src_info: &mut ObjectInfo,
src_opts: &ObjectOptions,
dst_opts: &ObjectOptions,
) -> Result<ObjectInfo>;
async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()>;
async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo>;
async fn delete_objects(
&self,
bucket: &str,
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> (Vec<DeletedObject>, Vec<Option<Error>>);
async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String>;
async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo>;
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()>;
async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>;
async fn restore_transitioned_object(self: Arc<Self>, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>;
pub trait ObjectOperations:
rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
impl<T> ObjectOperations for T where
T: rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
/// Listing and walking operations.
@@ -80,67 +94,40 @@ impl<T> ListOperations for T where
}
/// Multipart upload operations.
#[async_trait::async_trait]
#[allow(clippy::too_many_arguments)]
pub trait MultipartOperations: Send + Sync + Debug {
async fn list_multipart_uploads(
&self,
bucket: &str,
prefix: &str,
key_marker: Option<String>,
upload_id_marker: Option<String>,
delimiter: Option<String>,
max_uploads: usize,
) -> Result<ListMultipartsInfo>;
async fn new_multipart_upload(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<MultipartUploadResult>;
async fn copy_object_part(
&self,
src_bucket: &str,
src_object: &str,
dst_bucket: &str,
dst_object: &str,
upload_id: &str,
part_id: usize,
start_offset: i64,
length: i64,
src_info: &ObjectInfo,
src_opts: &ObjectOptions,
dst_opts: &ObjectOptions,
) -> Result<()>;
async fn put_object_part(
&self,
bucket: &str,
object: &str,
upload_id: &str,
part_id: usize,
data: &mut PutObjReader,
opts: &ObjectOptions,
) -> Result<PartInfo>;
async fn get_multipart_info(
&self,
bucket: &str,
object: &str,
upload_id: &str,
opts: &ObjectOptions,
) -> Result<MultipartInfo>;
async fn list_object_parts(
&self,
bucket: &str,
object: &str,
upload_id: &str,
part_number_marker: Option<usize>,
max_parts: usize,
opts: &ObjectOptions,
) -> Result<ListPartsInfo>;
async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, opts: &ObjectOptions) -> Result<()>;
async fn complete_multipart_upload(
self: Arc<Self>,
bucket: &str,
object: &str,
upload_id: &str,
uploaded_parts: Vec<CompletePart>,
opts: &ObjectOptions,
) -> Result<ObjectInfo>;
pub trait MultipartOperations:
rustfs_storage_api::MultipartOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
PutObjectReader = PutObjReader,
CompletePart = CompletePart,
ListMultipartsInfo = ListMultipartsInfo,
MultipartUploadResult = MultipartUploadResult,
PartInfo = PartInfo,
MultipartInfo = MultipartInfo,
ListPartsInfo = ListPartsInfo,
> + Send
+ Sync
+ Debug
{
}
impl<T> MultipartOperations for T where
T: rustfs_storage_api::MultipartOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
PutObjectReader = PutObjReader,
CompletePart = CompletePart,
ListMultipartsInfo = ListMultipartsInfo,
MultipartUploadResult = MultipartUploadResult,
PartInfo = PartInfo,
MultipartInfo = MultipartInfo,
ListPartsInfo = ListPartsInfo,
> + Send
+ Sync
+ Debug
{
}
/// Healing and repair operations.
+2 -4
View File
@@ -22,9 +22,7 @@ use crate::error::{
Error, Result, StorageError, is_all_not_found, is_all_volume_not_found, is_err_bucket_not_found, to_object_err,
};
use crate::set_disk::SetDisks;
use crate::store_api::{
ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectInfoOrErr, ObjectOperations, ObjectOptions, WalkOptions,
};
use crate::store_api::{ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectInfoOrErr, ObjectOptions, WalkOptions};
use crate::store_utils::is_reserved_or_invalid_bucket;
use crate::{store::ECStore, store_api::ListObjectsV2Info};
use futures::future::join_all;
@@ -33,7 +31,7 @@ use rustfs_filemeta::{
MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
merge_file_meta_versions,
};
use rustfs_storage_api::{VersionMarker, WalkVersionsSortOrder};
use rustfs_storage_api::{ObjectOperations as _, VersionMarker, WalkVersionsSortOrder};
use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
+1
View File
@@ -53,6 +53,7 @@ use crate::{
store_api::{ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
};
use rustfs_rio::HashReader;
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::S3ErrorCode;
+4 -2
View File
@@ -19,10 +19,12 @@ use rustfs_ecstore::{
disk::{DiskStore, endpoint::Endpoint},
error::StorageError,
store::ECStore,
store_api::{HealOperations, ObjectIO, ObjectOperations, ObjectOptions},
store_api::{HealOperations, ObjectOptions},
};
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::{BucketInfo, BucketOperations, DiskSetSelector, ListOperations as _, StorageAdminApi};
use rustfs_storage_api::{
BucketInfo, BucketOperations, DiskSetSelector, ListOperations as _, ObjectIO as _, ObjectOperations as _, StorageAdminApi,
};
use std::sync::Arc;
use tracing::{debug, error, warn};
+2 -2
View File
@@ -18,14 +18,14 @@ use rustfs_ecstore::{
disk::endpoint::Endpoint,
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
store::ECStore,
store_api::{ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
store_api::{ObjectOptions, PutObjReader},
};
use rustfs_heal::heal::{
manager::{HealConfig, HealManager},
storage::{ECStoreHealStorage, HealStorageAPI},
task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
};
use rustfs_storage_api::BucketOperations;
use rustfs_storage_api::{BucketOperations, ObjectIO as _, ObjectOperations as _};
use serial_test::serial;
use std::{
path::{Path, PathBuf},
+2 -2
View File
@@ -55,9 +55,9 @@ use super::{SwiftError, SwiftResult};
use axum::http::HeaderMap;
use rustfs_credentials::Credentials;
use rustfs_ecstore::resolve_object_store_handle;
use rustfs_ecstore::store_api::{ObjectIO, ObjectOperations, ObjectOptions, PutObjReader};
use rustfs_ecstore::store_api::{ObjectOptions, PutObjReader};
use rustfs_rio::HashReader;
use rustfs_storage_api::{BucketOperations, BucketOptions};
use rustfs_storage_api::{BucketOperations, BucketOptions, ObjectIO as _, ObjectOperations as _};
use std::collections::HashMap;
use tracing::debug;
use tracing::error;
+2 -2
View File
@@ -57,8 +57,8 @@ use super::object::{ObjectKeyMapper, head_object};
use super::{SwiftError, SwiftResult};
use rustfs_credentials::Credentials;
use rustfs_ecstore::resolve_object_store_handle;
use rustfs_ecstore::store_api::{ObjectOperations, ObjectOptions};
use rustfs_storage_api::ListOperations as _;
use rustfs_ecstore::store_api::ObjectOptions;
use rustfs_storage_api::{ListOperations as _, ObjectOperations as _};
use std::time::{SystemTime, UNIX_EPOCH};
use tracing::{debug, error};
+2 -2
View File
@@ -29,8 +29,8 @@ use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object
use rustfs_ecstore::resolve_object_store_handle;
use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
use rustfs_ecstore::store::ECStore;
use rustfs_ecstore::store_api::{GetObjectReader, ObjectIO, ObjectOperations, ObjectOptions};
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_ecstore::store_api::{GetObjectReader, ObjectOptions};
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
use s3s::S3Result;
use s3s::dto::SelectObjectContentInput;
use s3s::header::{
+10 -2
View File
@@ -1069,7 +1069,7 @@ pub async fn store_data_usage_in_backend(
#[cfg(test)]
mod tests {
use super::*;
use rustfs_ecstore::store_api::{GetObjectReader, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader};
use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use serial_test::serial;
use std::collections::HashMap;
use std::io::Cursor;
@@ -1119,7 +1119,15 @@ mod tests {
}
#[async_trait::async_trait]
impl ObjectIO for MemoryConfigStore {
impl rustfs_storage_api::ObjectIO for MemoryConfigStore {
type Error = rustfs_ecstore::error::Error;
type RangeSpec = rustfs_storage_api::HTTPRangeSpec;
type HeaderMap = http::HeaderMap;
type ObjectOptions = ObjectOptions;
type ObjectInfo = ObjectInfo;
type GetObjectReader = GetObjectReader;
type PutObjectReader = PutObjReader;
async fn get_object_reader(
&self,
bucket: &str,
@@ -28,7 +28,7 @@ use rustfs_ecstore::{
global::GLOBAL_TierConfigMgr,
pools::path2_bucket_object_with_base_path,
store::ECStore,
store_api::{MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
store_api::{ObjectOptions, PutObjReader},
tier::{
tier_config::{TierConfig, TierMinIO, TierType},
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
@@ -38,8 +38,9 @@ use rustfs_filemeta::FileMeta;
use rustfs_scanner::scanner::init_data_scanner;
use rustfs_scanner::scanner_folder::ScannerItem;
use rustfs_scanner::scanner_io::ScannerIODisk;
use rustfs_storage_api::ListOperations as _;
use rustfs_storage_api::{BucketOperations, MakeBucketOptions};
use rustfs_storage_api::{
BucketOperations, ListOperations as _, MakeBucketOptions, MultipartOperations as _, ObjectIO as _, ObjectOperations as _,
};
use rustfs_utils::path::path_join_buf;
use s3s::dto::RestoreRequest;
use serial_test::serial;
+1
View File
@@ -26,5 +26,6 @@ pub use error::{StorageErrorCode, StorageResult};
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};
pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};
pub use object::{MultipartOperations, ObjectIO, ObjectOperations};
pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};
pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};
+185
View File
@@ -180,6 +180,106 @@ impl<Filter> Default for WalkOptions<Filter> {
}
}
#[async_trait::async_trait]
pub trait ObjectIO: Send + Sync + fmt::Debug + 'static {
type Error: std::error::Error + Send + Sync + 'static;
type RangeSpec: Send + 'static;
type HeaderMap: Send + 'static;
type ObjectOptions: Send + Sync + 'static;
type ObjectInfo: Send + 'static;
type GetObjectReader: Send + 'static;
type PutObjectReader: Send + 'static;
async fn get_object_reader(
&self,
bucket: &str,
object: &str,
range: Option<Self::RangeSpec>,
h: Self::HeaderMap,
opts: &Self::ObjectOptions,
) -> Result<Self::GetObjectReader, Self::Error>;
async fn put_object(
&self,
bucket: &str,
object: &str,
data: &mut Self::PutObjectReader,
opts: &Self::ObjectOptions,
) -> Result<Self::ObjectInfo, Self::Error>;
}
#[async_trait::async_trait]
#[allow(clippy::too_many_arguments)]
pub trait ObjectOperations: Send + Sync + fmt::Debug {
type Error: std::error::Error + Send + Sync + 'static;
type ObjectInfo: Send + 'static;
type ObjectOptions: Send + Sync + 'static;
type FileInfo: Send + Sync + 'static;
type ObjectToDelete: Send + 'static;
type DeletedObject: Send + 'static;
async fn get_object_info(
&self,
bucket: &str,
object: &str,
opts: &Self::ObjectOptions,
) -> Result<Self::ObjectInfo, Self::Error>;
async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &Self::ObjectOptions) -> Result<(), Self::Error>;
async fn copy_object(
&self,
src_bucket: &str,
src_object: &str,
dst_bucket: &str,
dst_object: &str,
src_info: &mut Self::ObjectInfo,
src_opts: &Self::ObjectOptions,
dst_opts: &Self::ObjectOptions,
) -> Result<Self::ObjectInfo, Self::Error>;
async fn delete_object_version(
&self,
bucket: &str,
object: &str,
fi: &Self::FileInfo,
force_del_marker: bool,
) -> Result<(), Self::Error>;
async fn delete_object(&self, bucket: &str, object: &str, opts: Self::ObjectOptions)
-> Result<Self::ObjectInfo, Self::Error>;
async fn delete_objects(
&self,
bucket: &str,
objects: Vec<Self::ObjectToDelete>,
opts: Self::ObjectOptions,
) -> (Vec<Self::DeletedObject>, Vec<Option<Self::Error>>);
async fn put_object_metadata(
&self,
bucket: &str,
object: &str,
opts: &Self::ObjectOptions,
) -> Result<Self::ObjectInfo, Self::Error>;
async fn get_object_tags(&self, bucket: &str, object: &str, opts: &Self::ObjectOptions) -> Result<String, Self::Error>;
async fn put_object_tags(
&self,
bucket: &str,
object: &str,
tags: &str,
opts: &Self::ObjectOptions,
) -> Result<Self::ObjectInfo, Self::Error>;
async fn delete_object_tags(
&self,
bucket: &str,
object: &str,
opts: &Self::ObjectOptions,
) -> Result<Self::ObjectInfo, Self::Error>;
async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<(), Self::Error>;
async fn transition_object(&self, bucket: &str, object: &str, opts: &Self::ObjectOptions) -> Result<(), Self::Error>;
async fn restore_transitioned_object(
self: Arc<Self>,
bucket: &str,
object: &str,
opts: &Self::ObjectOptions,
) -> Result<(), Self::Error>;
}
#[async_trait::async_trait]
#[allow(clippy::too_many_arguments)]
pub trait ListOperations: Send + Sync + fmt::Debug {
@@ -223,6 +323,91 @@ pub trait ListOperations: Send + Sync + fmt::Debug {
) -> Result<(), Self::Error>;
}
#[async_trait::async_trait]
#[allow(clippy::too_many_arguments)]
pub trait MultipartOperations: Send + Sync + fmt::Debug {
type Error: std::error::Error + Send + Sync + 'static;
type ObjectInfo: Send + 'static;
type ObjectOptions: Send + Sync + 'static;
type PutObjectReader: Send + 'static;
type CompletePart: Send + 'static;
type ListMultipartsInfo: Send + 'static;
type MultipartUploadResult: Send + 'static;
type PartInfo: Send + 'static;
type MultipartInfo: Send + 'static;
type ListPartsInfo: Send + 'static;
async fn list_multipart_uploads(
&self,
bucket: &str,
prefix: &str,
key_marker: Option<String>,
upload_id_marker: Option<String>,
delimiter: Option<String>,
max_uploads: usize,
) -> Result<Self::ListMultipartsInfo, Self::Error>;
async fn new_multipart_upload(
&self,
bucket: &str,
object: &str,
opts: &Self::ObjectOptions,
) -> Result<Self::MultipartUploadResult, Self::Error>;
async fn copy_object_part(
&self,
src_bucket: &str,
src_object: &str,
dst_bucket: &str,
dst_object: &str,
upload_id: &str,
part_id: usize,
start_offset: i64,
length: i64,
src_info: &Self::ObjectInfo,
src_opts: &Self::ObjectOptions,
dst_opts: &Self::ObjectOptions,
) -> Result<(), Self::Error>;
async fn put_object_part(
&self,
bucket: &str,
object: &str,
upload_id: &str,
part_id: usize,
data: &mut Self::PutObjectReader,
opts: &Self::ObjectOptions,
) -> Result<Self::PartInfo, Self::Error>;
async fn get_multipart_info(
&self,
bucket: &str,
object: &str,
upload_id: &str,
opts: &Self::ObjectOptions,
) -> Result<Self::MultipartInfo, Self::Error>;
async fn list_object_parts(
&self,
bucket: &str,
object: &str,
upload_id: &str,
part_number_marker: Option<usize>,
max_parts: usize,
opts: &Self::ObjectOptions,
) -> Result<Self::ListPartsInfo, Self::Error>;
async fn abort_multipart_upload(
&self,
bucket: &str,
object: &str,
upload_id: &str,
opts: &Self::ObjectOptions,
) -> Result<(), Self::Error>;
async fn complete_multipart_upload(
self: Arc<Self>,
bucket: &str,
object: &str,
upload_id: &str,
uploaded_parts: Vec<Self::CompletePart>,
opts: &Self::ObjectOptions,
) -> Result<Self::ObjectInfo, Self::Error>;
}
#[derive(Debug, Default)]
pub struct ListObjectsInfo<ObjectItem> {
pub is_truncated: bool,