refactor: continue storage api contract cleanup (#3580)

* refactor: move delete object contracts to storage api

* refactor: narrow store api compatibility exports

* refactor: route table catalog test through storage compat
This commit is contained in:
安正超
2026-06-18 22:42:02 +08:00
committed by GitHub
parent 80144ec886
commit c28fee0013
24 changed files with 310 additions and 113 deletions
@@ -37,7 +37,7 @@ use crate::global::GLOBAL_LocalNodeName;
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks};
use crate::store::ECStore;
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, ObjectToDelete};
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::tier::warm_backend::WarmBackendGetOpts;
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
use futures::Future;
@@ -58,7 +58,9 @@ use rustfs_filemeta::{
VersionPurgeStatusType, get_file_info, is_restored_object_on_disk,
};
use rustfs_s3_types::EventName;
use rustfs_storage_api::{HTTPRangeSpec, ListOperations as _, MultipartOperations as _, ObjectOperations as _};
use rustfs_storage_api::{
DeletedObject, HTTPRangeSpec, ListOperations as _, MultipartOperations as _, ObjectOperations as _, ObjectToDelete,
};
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest,
@@ -2692,9 +2694,9 @@ pub async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &
expiry_state.enqueue_by_days(oi, event, src).await
}
fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> crate::store_api::DeletedObject {
fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> DeletedObject {
if dobj.delete_marker {
return crate::store_api::DeletedObject {
return DeletedObject {
object_name: oi.name.clone(),
delete_marker: true,
delete_marker_version_id: dobj.version_id,
@@ -2704,7 +2706,7 @@ fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> crate::store_
}
if oi.delete_marker && oi.version_id.is_some() {
return crate::store_api::DeletedObject {
return DeletedObject {
object_name: oi.name.clone(),
delete_marker: false,
delete_marker_version_id: oi.version_id,
@@ -2713,7 +2715,7 @@ fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> crate::store_
};
}
crate::store_api::DeletedObject {
DeletedObject {
object_name: oi.name.clone(),
delete_marker: false,
version_id: oi.version_id,
@@ -42,6 +42,7 @@ use rustfs_filemeta::VersionPurgeStatusType;
use rustfs_filemeta::replication_statuses_map;
use rustfs_filemeta::version_purge_statuses_map;
use rustfs_filemeta::{REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE};
use rustfs_storage_api::DeletedObject;
use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str};
use std::any::Any;
use std::sync::Arc;
@@ -838,7 +839,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
// get_object_info here because the delete-marker or version may
// already be absent from the local store — that is expected.
let dv = DeletedObjectReplicationInfo {
delete_object: crate::store_api::DeletedObject {
delete_object: DeletedObject {
object_name: entry.object.clone(),
version_id: entry.version_id,
delete_marker_version_id: entry.delete_marker_version_id,
@@ -1577,7 +1578,7 @@ pub async fn queue_replication_heal_internal(
};
let dv = DeletedObjectReplicationInfo {
delete_object: crate::store_api::DeletedObject {
delete_object: DeletedObject {
object_name: roi.name.clone(),
delete_marker_version_id: dm_version_id,
version_id,
@@ -33,10 +33,7 @@ use crate::global::GLOBAL_LocalNodeName;
use crate::global::get_global_bucket_monitor;
use crate::resolve_object_store_handle;
use crate::set_disk::get_lock_acquire_timeout;
use crate::store_api::{
DeletedObject, ListOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete,
WalkOptions,
};
use crate::store_api::{ListOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, WalkOptions};
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput};
use aws_sdk_s3::primitives::ByteStream;
@@ -61,7 +58,7 @@ use rustfs_filemeta::{
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
};
use rustfs_s3_types::EventName;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectToDelete};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
@@ -25,10 +25,10 @@ use crate::bucket::replication::{DeletedObjectReplicationInfo, check_replicate_d
use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::store::ECStore;
use crate::store_api::{ObjectOptions, ObjectToDelete};
use crate::store_api::ObjectOptions;
use rustfs_filemeta::{REPLICATE_INCOMING_DELETE, ReplicationState, version_purge_statuses_map};
use rustfs_lock::MAX_DELETE_LIST;
use rustfs_storage_api::ObjectOperations as _;
use rustfs_storage_api::{ObjectOperations as _, ObjectToDelete};
fn lifecycle_version_delete_replication_state(
replicate_decision_str: String,
+4 -7
View File
@@ -34,8 +34,8 @@ use crate::error::{Error, Result, is_err_version_not_found};
use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr};
use crate::store_api::ListObjectVersionsInfo;
use crate::store_api::ObjectOptions;
use crate::store_api::{ObjectInfoOrErr, WalkOptions};
use crate::store_api::{ObjectOptions, ObjectToDelete};
use crate::{
bucket::lifecycle::bucket_lifecycle_ops::{
LifecycleOps, gen_transition_objname, get_transitioned_object_reader, put_restore_opts,
@@ -51,10 +51,7 @@ use crate::{
// event::name::EventName,
event_notification::{EventArgs, send_event},
global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure},
store_api::{
DeletedObject, GetObjectReader, ListObjectsV2Info, MultipartOperations, ObjectIO, ObjectInfo, ObjectOperations,
PutObjReader,
},
store_api::{GetObjectReader, ListObjectsV2Info, MultipartOperations, ObjectIO, ObjectInfo, ObjectOperations, PutObjReader},
store_init::load_format_erasure,
};
use bytes::Bytes;
@@ -87,8 +84,8 @@ use rustfs_object_capacity::capacity_scope::{
use rustfs_s3_types::EventName;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo,
MakeBucketOptions, MultipartInfo, MultipartUploadResult, PartInfo,
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
};
use rustfs_storage_api::{MultipartOperations as _, NamespaceLocking as _, ObjectIO as _, ObjectOperations as _};
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
+3 -6
View File
@@ -27,10 +27,7 @@ use crate::{
error::StorageError,
global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure},
set_disk::SetDisks,
store_api::{
DeletedObject, GetObjectReader, ListObjectVersionsInfo, ListObjectsV2Info, ObjectInfo, ObjectOptions, ObjectToDelete,
PutObjReader,
},
store_api::{GetObjectReader, ListObjectVersionsInfo, ListObjectsV2Info, ObjectInfo, ObjectOptions, PutObjReader},
store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
};
use futures::{
@@ -50,8 +47,8 @@ use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_storage_api::CompletePart;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo, MakeBucketOptions,
MultipartInfo, MultipartUploadResult, PartInfo,
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DeletedObject, ListMultipartsInfo, ListPartsInfo,
MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
};
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
+3 -3
View File
@@ -62,7 +62,7 @@ use crate::{
endpoints::EndpointServerPools,
rpc::S3PeerSys,
sets::Sets,
store_api::{DeletedObject, GetObjectReader, ListObjectsV2Info, ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader},
store_api::{GetObjectReader, ListObjectsV2Info, ObjectInfo, ObjectOptions, PutObjReader},
store_init,
};
use futures::future::join_all;
@@ -77,8 +77,8 @@ use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo,
MakeBucketOptions, MultipartInfo, MultipartUploadResult, PartInfo,
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
};
use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf};
use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
+4 -1
View File
@@ -1,5 +1,8 @@
use super::*;
use rustfs_storage_api::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
use rustfs_storage_api::{
CompletePart, DeletedObject, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, ObjectToDelete,
PartInfo,
};
pub trait ObjectIO:
rustfs_storage_api::ObjectIO<
+2 -53
View File
@@ -9,6 +9,8 @@ pub type ListObjectsV2Info = rustfs_storage_api::ListObjectsV2Info<ObjectInfo>;
pub type ListObjectVersionsInfo = rustfs_storage_api::ListObjectVersionsInfo<ObjectInfo>;
pub type ObjectInfoOrErr = rustfs_storage_api::ObjectInfoOrErr<ObjectInfo, Error>;
pub type WalkOptions = rustfs_storage_api::WalkOptions<WalkFilter>;
pub type ObjectToDelete = rustfs_storage_api::ObjectToDelete;
pub type DeletedObject = rustfs_storage_api::DeletedObject;
#[derive(Debug, Default, Clone)]
pub struct ObjectOptions {
@@ -714,59 +716,6 @@ fn versions_after_marker(file_infos: &rustfs_filemeta::FileInfoVersions, marker:
.unwrap_or(&file_infos.versions)
}
#[derive(Debug, Default, Clone)]
pub struct ObjectToDelete {
pub object_name: String,
pub version_id: Option<Uuid>,
pub delete_marker_replication_status: Option<String>,
pub version_purge_status: Option<VersionPurgeStatusType>,
pub version_purge_statuses: Option<String>,
pub replicate_decision_str: Option<String>,
}
impl ObjectToDelete {
pub fn replication_state(&self) -> ReplicationState {
ReplicationState {
replication_status_internal: self.delete_marker_replication_status.clone(),
version_purge_status_internal: self.version_purge_statuses.clone(),
replicate_decision_str: self.replicate_decision_str.clone().unwrap_or_default(),
targets: replication_statuses_map(self.delete_marker_replication_status.as_deref().unwrap_or_default()),
purge_targets: version_purge_statuses_map(self.version_purge_statuses.as_deref().unwrap_or_default()),
..Default::default()
}
}
}
#[derive(Debug, Default, Clone)]
pub struct DeletedObject {
pub delete_marker: bool,
pub delete_marker_version_id: Option<Uuid>,
pub object_name: String,
pub version_id: Option<Uuid>,
// MTime of DeleteMarker on source that needs to be propagated to replica
pub delete_marker_mtime: Option<OffsetDateTime>,
// to support delete marker replication
pub replication_state: Option<ReplicationState>,
pub found: bool,
pub force_delete: bool,
}
impl DeletedObject {
pub fn version_purge_status(&self) -> VersionPurgeStatusType {
self.replication_state
.as_ref()
.map(|v| v.composite_version_purge_status())
.unwrap_or(VersionPurgeStatusType::Empty)
}
pub fn delete_marker_replication_status(&self) -> ReplicationStatusType {
self.replication_state
.as_ref()
.map(|v| v.composite_replication_status())
.unwrap_or(ReplicationStatusType::Empty)
}
}
type WalkFilter = fn(&FileInfo) -> bool;
#[cfg(test)]
+5 -1
View File
@@ -13,7 +13,11 @@
// limitations under the License.
pub(crate) mod ecstore {
pub(crate) use rustfs_ecstore::{data_usage, disk, error, global, store, store_api};
pub(crate) use rustfs_ecstore::{data_usage, disk, error, global, store};
pub(crate) mod store_api {
pub(crate) use rustfs_ecstore::store_api::{ObjectInfo, ObjectOptions, PutObjReader};
}
}
pub(crate) use self::ecstore::{
data_usage::DATA_USAGE_CACHE_NAME,
+3 -1
View File
@@ -13,7 +13,9 @@
// limitations under the License.
mod ecstore {
pub(super) use rustfs_ecstore::store_api;
pub(super) mod store_api {
pub(crate) use rustfs_ecstore::store_api::{ObjectInfo, ObjectOptions};
}
}
pub(super) type IamObjectInfo = ecstore::store_api::ObjectInfo;
+5 -1
View File
@@ -13,7 +13,11 @@
// limitations under the License.
pub(crate) mod ecstore {
pub(crate) use rustfs_ecstore::{config, global, store_api};
pub(crate) use rustfs_ecstore::{config, global};
pub(crate) mod store_api {
pub(crate) use rustfs_ecstore::store_api::ObjectInfo;
}
}
pub type NotifyObjectInfo = ecstore::store_api::ObjectInfo;
+5 -1
View File
@@ -13,7 +13,11 @@
// limitations under the License.
mod ecstore {
pub(super) use rustfs_ecstore::{bucket, error, resolve_object_store_handle, store, store_api};
pub(super) use rustfs_ecstore::{bucket, error, resolve_object_store_handle, store};
pub(super) mod store_api {
pub(crate) use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
}
}
use self::ecstore::{
bucket::{metadata::BucketMetadata, metadata_sys},
+5 -1
View File
@@ -13,7 +13,11 @@
// limitations under the License.
pub(crate) mod ecstore {
pub(crate) use rustfs_ecstore::{error, resolve_object_store_handle, set_disk, store, store_api};
pub(crate) use rustfs_ecstore::{error, resolve_object_store_handle, set_disk, store};
pub(crate) mod store_api {
pub(crate) use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
}
}
use ecstore::store_api::{
+8 -4
View File
@@ -16,8 +16,12 @@ use http::HeaderMap;
pub(crate) mod ecstore {
pub(crate) use rustfs_ecstore::{
bucket, cache_value, config, data_usage, disk, error, global, pools, resolve_object_store_handle, set_disk, store,
store_api, store_utils,
store_utils,
};
pub(crate) mod store_api {
pub(crate) use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
}
}
pub(crate) use self::ecstore::{
bucket::{
@@ -49,11 +53,11 @@ pub(crate) use self::ecstore::{
store::ECStore,
store_api::{
GetObjectReader as EcstoreGetObjectReader, ObjectInfo as EcstoreObjectInfo, ObjectOptions as EcstoreObjectOptions,
ObjectToDelete as EcstoreObjectToDelete, PutObjReader as EcstorePutObjReader,
PutObjReader as EcstorePutObjReader,
},
store_utils::is_reserved_or_invalid_bucket,
};
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO};
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO, ObjectToDelete};
use std::sync::Arc;
#[cfg(test)]
@@ -65,7 +69,7 @@ pub(crate) use self::ecstore::{
pub type ScannerGetObjectReader = EcstoreGetObjectReader;
pub type ScannerObjectInfo = EcstoreObjectInfo;
pub type ScannerObjectOptions = EcstoreObjectOptions;
pub type ScannerObjectToDelete = EcstoreObjectToDelete;
pub type ScannerObjectToDelete = ObjectToDelete;
pub type ScannerPutObjReader = EcstorePutObjReader;
pub(crate) fn resolve_scanner_object_store_handle() -> Option<Arc<ECStore>> {
+1
View File
@@ -29,6 +29,7 @@ doctest = false
[dependencies]
async-trait.workspace = true
rustfs-filemeta.workspace = true
serde.workspace = true
time.workspace = true
uuid.workspace = true
+1
View File
@@ -24,6 +24,7 @@ pub use admin::{DiskSetSelector, StorageAdminApi};
pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
pub use error::{StorageErrorCode, StorageResult};
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
pub use object::{DeletedObject, ObjectToDelete};
pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};
pub use object::{HealOperations, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations};
pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};
+83
View File
@@ -14,6 +14,10 @@
use std::fmt;
use std::sync::Arc;
use rustfs_filemeta::{
ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_statuses_map, version_purge_statuses_map,
};
use time::OffsetDateTime;
use uuid::Uuid;
@@ -155,6 +159,57 @@ pub enum WalkVersionsSortOrder {
Descending,
}
#[derive(Debug, Default, Clone)]
pub struct ObjectToDelete {
pub object_name: String,
pub version_id: Option<Uuid>,
pub delete_marker_replication_status: Option<String>,
pub version_purge_status: Option<VersionPurgeStatusType>,
pub version_purge_statuses: Option<String>,
pub replicate_decision_str: Option<String>,
}
impl ObjectToDelete {
pub fn replication_state(&self) -> ReplicationState {
ReplicationState {
replication_status_internal: self.delete_marker_replication_status.clone(),
version_purge_status_internal: self.version_purge_statuses.clone(),
replicate_decision_str: self.replicate_decision_str.clone().unwrap_or_default(),
targets: replication_statuses_map(self.delete_marker_replication_status.as_deref().unwrap_or_default()),
purge_targets: version_purge_statuses_map(self.version_purge_statuses.as_deref().unwrap_or_default()),
..Default::default()
}
}
}
#[derive(Debug, Default, Clone)]
pub struct DeletedObject {
pub delete_marker: bool,
pub delete_marker_version_id: Option<Uuid>,
pub object_name: String,
pub version_id: Option<Uuid>,
pub delete_marker_mtime: Option<OffsetDateTime>,
pub replication_state: Option<ReplicationState>,
pub found: bool,
pub force_delete: bool,
}
impl DeletedObject {
pub fn version_purge_status(&self) -> VersionPurgeStatusType {
self.replication_state
.as_ref()
.map(|v| v.composite_version_purge_status())
.unwrap_or(VersionPurgeStatusType::Empty)
}
pub fn delete_marker_replication_status(&self) -> ReplicationStatusType {
self.replication_state
.as_ref()
.map(|v| v.composite_replication_status())
.unwrap_or(ReplicationStatusType::Empty)
}
}
#[derive(Clone)]
pub struct WalkOptions<Filter> {
pub filter: Option<Filter>,
@@ -714,6 +769,34 @@ mod tests {
assert!(!opts.include_free_versions);
}
#[test]
fn object_to_delete_builds_replication_state() {
let object = ObjectToDelete {
object_name: "photo.jpg".to_owned(),
delete_marker_replication_status: Some("PENDING".to_owned()),
version_purge_statuses: Some("arn:minio:replication:::target=PENDING".to_owned()),
replicate_decision_str: Some("arn:minio:replication:::target=PENDING".to_owned()),
..Default::default()
};
let state = object.replication_state();
assert_eq!(state.replication_status_internal.as_deref(), Some("PENDING"));
assert_eq!(
state.version_purge_status_internal.as_deref(),
Some("arn:minio:replication:::target=PENDING")
);
assert_eq!(state.replicate_decision_str, "arn:minio:replication:::target=PENDING");
}
#[test]
fn deleted_object_status_helpers_default_to_empty() {
let object = DeletedObject::default();
assert_eq!(object.version_purge_status(), VersionPurgeStatusType::Empty);
assert_eq!(object.delete_marker_replication_status(), ReplicationStatusType::Empty);
}
#[derive(Debug)]
struct TestListBackend;