mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
refactor: move list operations contract (#3550)
This commit is contained in:
@@ -37,9 +37,7 @@ use crate::global::GLOBAL_LocalNodeName;
|
||||
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
|
||||
use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks};
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{
|
||||
GetObjectReader, ListOperations, MultipartOperations, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete,
|
||||
};
|
||||
use crate::store_api::{GetObjectReader, MultipartOperations, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete};
|
||||
use crate::tier::warm_backend::WarmBackendGetOpts;
|
||||
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
|
||||
use futures::Future;
|
||||
@@ -60,7 +58,7 @@ use rustfs_filemeta::{
|
||||
VersionPurgeStatusType, get_file_info, is_restored_object_on_disk,
|
||||
};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
use rustfs_storage_api::{HTTPRangeSpec, ListOperations as _};
|
||||
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest,
|
||||
|
||||
@@ -24,7 +24,8 @@ use crate::config::com::{delete_config, read_config, save_config};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{ListOperations, ObjectIO, ObjectOperations};
|
||||
use crate::store_api::{ObjectIO, ObjectOperations};
|
||||
use rustfs_storage_api::ListOperations as _;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
|
||||
|
||||
@@ -20,8 +20,8 @@ use tokio_util::sync::CancellationToken;
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::Result;
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{ListOperations, ObjectInfo, ObjectInfoOrErr, WalkOptions};
|
||||
use rustfs_storage_api::{BucketOperations, BucketOptions};
|
||||
use crate::store_api::{ObjectInfo, ObjectInfoOrErr, WalkOptions};
|
||||
use rustfs_storage_api::{BucketOperations, BucketOptions, ListOperations as _};
|
||||
|
||||
pub const DEFAULT_FREE_VERSION_RECOVERY_LIMIT: usize = 1_000;
|
||||
const DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT: usize = 10_000;
|
||||
|
||||
@@ -20,7 +20,6 @@ use crate::{
|
||||
disk::DiskAPI,
|
||||
error::{Error, classify_system_path_failure_reason},
|
||||
store::ECStore,
|
||||
store_api::ListOperations,
|
||||
};
|
||||
pub use local_snapshot::{
|
||||
DATA_USAGE_DIR, DATA_USAGE_STATE_DIR, LOCAL_USAGE_SNAPSHOT_VERSION, LocalUsageSnapshot, LocalUsageSnapshotMeta,
|
||||
@@ -31,6 +30,7 @@ use rustfs_data_usage::{
|
||||
BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary,
|
||||
};
|
||||
use rustfs_io_metrics::record_system_path_failure;
|
||||
use rustfs_storage_api::ListOperations as _;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use std::{
|
||||
collections::{HashMap, HashSet, hash_map::Entry},
|
||||
|
||||
@@ -52,8 +52,8 @@ use crate::{
|
||||
event_notification::{EventArgs, send_event},
|
||||
global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure},
|
||||
store_api::{
|
||||
DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, ListOperations, MultipartOperations, NamespaceLocking,
|
||||
ObjectIO, ObjectInfo, ObjectOperations, PutObjReader,
|
||||
DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, MultipartOperations, NamespaceLocking, ObjectIO,
|
||||
ObjectInfo, ObjectOperations, PutObjReader,
|
||||
},
|
||||
store_init::load_format_erasure,
|
||||
};
|
||||
@@ -2985,7 +2985,15 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ListOperations for SetDisks {
|
||||
impl rustfs_storage_api::ListOperations for SetDisks {
|
||||
type Error = Error;
|
||||
type ListObjectsV2Info = ListObjectsV2Info;
|
||||
type ListObjectVersionsInfo = ListObjectVersionsInfo;
|
||||
type ObjectInfoOrErr = ObjectInfoOrErr;
|
||||
type WalkOptions = WalkOptions;
|
||||
type WalkCancellation = CancellationToken;
|
||||
type WalkResultSender = Sender<ObjectInfoOrErr>;
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn list_objects_v2(
|
||||
self: Arc<Self>,
|
||||
|
||||
@@ -28,9 +28,8 @@ use crate::{
|
||||
global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure},
|
||||
set_disk::SetDisks,
|
||||
store_api::{
|
||||
DeletedObject, GetObjectReader, HealOperations, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations,
|
||||
MultipartOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete,
|
||||
PutObjReader,
|
||||
DeletedObject, GetObjectReader, HealOperations, ListObjectVersionsInfo, ListObjectsV2Info, MultipartOperations,
|
||||
NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader,
|
||||
},
|
||||
store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
|
||||
};
|
||||
@@ -647,7 +646,15 @@ impl ObjectOperations for Sets {
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ListOperations for Sets {
|
||||
impl rustfs_storage_api::ListOperations for Sets {
|
||||
type Error = Error;
|
||||
type ListObjectsV2Info = ListObjectsV2Info;
|
||||
type ListObjectVersionsInfo = ListObjectVersionsInfo;
|
||||
type ObjectInfoOrErr = ObjectInfoOrErr;
|
||||
type WalkOptions = WalkOptions;
|
||||
type WalkCancellation = CancellationToken;
|
||||
type WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>;
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn list_objects_v2(
|
||||
self: Arc<Self>,
|
||||
|
||||
@@ -63,8 +63,8 @@ use crate::{
|
||||
rpc::S3PeerSys,
|
||||
sets::Sets,
|
||||
store_api::{
|
||||
DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, ListOperations, MultipartOperations, NamespaceLocking,
|
||||
ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader,
|
||||
DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, MultipartOperations, NamespaceLocking, ObjectInfo,
|
||||
ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader,
|
||||
},
|
||||
store_init,
|
||||
};
|
||||
@@ -504,7 +504,15 @@ impl ObjectOperations for ECStore {
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ListOperations for ECStore {
|
||||
impl rustfs_storage_api::ListOperations for ECStore {
|
||||
type Error = Error;
|
||||
type ListObjectsV2Info = ListObjectsV2Info;
|
||||
type ListObjectVersionsInfo = ListObjectVersionsInfo;
|
||||
type ObjectInfoOrErr = ObjectInfoOrErr;
|
||||
type WalkOptions = WalkOptions;
|
||||
type WalkCancellation = CancellationToken;
|
||||
type WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>;
|
||||
|
||||
// @continuation_token marker
|
||||
// @start_after as marker when continuation_token empty
|
||||
// @delimiter default="/", empty when recursive
|
||||
|
||||
@@ -49,39 +49,34 @@ pub trait ObjectOperations: Send + Sync + Debug {
|
||||
}
|
||||
|
||||
/// Listing and walking operations.
|
||||
#[async_trait::async_trait]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub trait ListOperations: Send + Sync + Debug {
|
||||
async fn list_objects_v2(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<String>,
|
||||
delimiter: Option<String>,
|
||||
max_keys: i32,
|
||||
fetch_owner: bool,
|
||||
start_after: Option<String>,
|
||||
incl_deleted: bool,
|
||||
) -> Result<ListObjectsV2Info>;
|
||||
pub trait ListOperations:
|
||||
rustfs_storage_api::ListOperations<
|
||||
Error = Error,
|
||||
ListObjectsV2Info = ListObjectsV2Info,
|
||||
ListObjectVersionsInfo = ListObjectVersionsInfo,
|
||||
ObjectInfoOrErr = ObjectInfoOrErr,
|
||||
WalkOptions = WalkOptions,
|
||||
WalkCancellation = CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
{
|
||||
}
|
||||
|
||||
async fn list_object_versions(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
marker: Option<String>,
|
||||
version_marker: Option<String>,
|
||||
delimiter: Option<String>,
|
||||
max_keys: i32,
|
||||
) -> Result<ListObjectVersionsInfo>;
|
||||
|
||||
async fn walk(
|
||||
self: Arc<Self>,
|
||||
rx: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
|
||||
opts: WalkOptions,
|
||||
) -> Result<()>;
|
||||
impl<T> ListOperations for T where
|
||||
T: rustfs_storage_api::ListOperations<
|
||||
Error = Error,
|
||||
ListObjectsV2Info = ListObjectsV2Info,
|
||||
ListObjectVersionsInfo = ListObjectVersionsInfo,
|
||||
ObjectInfoOrErr = ObjectInfoOrErr,
|
||||
WalkOptions = WalkOptions,
|
||||
WalkCancellation = CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
{
|
||||
}
|
||||
|
||||
/// Multipart upload operations.
|
||||
|
||||
@@ -19,10 +19,10 @@ use rustfs_ecstore::{
|
||||
disk::{DiskStore, endpoint::Endpoint},
|
||||
error::StorageError,
|
||||
store::ECStore,
|
||||
store_api::{HealOperations, ListOperations, ObjectIO, ObjectOperations, ObjectOptions},
|
||||
store_api::{HealOperations, ObjectIO, ObjectOperations, ObjectOptions},
|
||||
};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_storage_api::{BucketInfo, BucketOperations, DiskSetSelector, StorageAdminApi};
|
||||
use rustfs_storage_api::{BucketInfo, BucketOperations, DiskSetSelector, ListOperations as _, StorageAdminApi};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ use crate::{
|
||||
use futures::future::join_all;
|
||||
use rustfs_credentials::get_global_action_cred;
|
||||
use rustfs_ecstore::error::{StorageError, classify_system_path_failure_reason};
|
||||
use rustfs_ecstore::store_api::{ListOperations as _, ObjectInfoOrErr, WalkOptions};
|
||||
use rustfs_ecstore::store_api::{ObjectInfoOrErr, WalkOptions};
|
||||
use rustfs_ecstore::{
|
||||
config::{
|
||||
RUSTFS_CONFIG_PREFIX,
|
||||
@@ -35,6 +35,7 @@ use rustfs_ecstore::{
|
||||
use rustfs_io_metrics::record_system_path_failure;
|
||||
use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
|
||||
use rustfs_storage_api::HTTPPreconditions;
|
||||
use rustfs_storage_api::ListOperations as _;
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::sync::{LazyLock, Mutex};
|
||||
|
||||
@@ -21,8 +21,9 @@ use super::types::Container;
|
||||
use super::{SwiftError, SwiftResult};
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_ecstore::resolve_object_store_handle;
|
||||
use rustfs_ecstore::store_api::ListOperations;
|
||||
use rustfs_storage_api::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions};
|
||||
use rustfs_storage_api::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListOperations as _, MakeBucketOptions,
|
||||
};
|
||||
use s3s::dto::{Tag, Tagging};
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::{debug, error};
|
||||
|
||||
@@ -57,7 +57,8 @@ use super::object::{ObjectKeyMapper, head_object};
|
||||
use super::{SwiftError, SwiftResult};
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_ecstore::resolve_object_store_handle;
|
||||
use rustfs_ecstore::store_api::{ListOperations, ObjectOperations, ObjectOptions};
|
||||
use rustfs_ecstore::store_api::{ObjectOperations, ObjectOptions};
|
||||
use rustfs_storage_api::ListOperations as _;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tracing::{debug, error};
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ use rustfs_ecstore::{
|
||||
global::GLOBAL_TierConfigMgr,
|
||||
pools::path2_bucket_object_with_base_path,
|
||||
store::ECStore,
|
||||
store_api::{ListOperations, MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
|
||||
store_api::{MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
|
||||
tier::{
|
||||
tier_config::{TierConfig, TierMinIO, TierType},
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
@@ -38,6 +38,7 @@ use rustfs_filemeta::FileMeta;
|
||||
use rustfs_scanner::scanner::init_data_scanner;
|
||||
use rustfs_scanner::scanner_folder::ScannerItem;
|
||||
use rustfs_scanner::scanner_io::ScannerIODisk;
|
||||
use rustfs_storage_api::ListOperations as _;
|
||||
use rustfs_storage_api::{BucketOperations, MakeBucketOptions};
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use s3s::dto::RestoreRequest;
|
||||
|
||||
@@ -25,6 +25,6 @@ pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOption
|
||||
pub use error::{StorageErrorCode, StorageResult};
|
||||
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
|
||||
pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};
|
||||
pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ObjectInfoOrErr};
|
||||
pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};
|
||||
pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};
|
||||
pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -179,6 +180,49 @@ impl<Filter> Default for WalkOptions<Filter> {
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub trait ListOperations: Send + Sync + fmt::Debug {
|
||||
type Error: std::error::Error + Send + Sync + 'static;
|
||||
type ListObjectsV2Info: Send + 'static;
|
||||
type ListObjectVersionsInfo: Send + 'static;
|
||||
type ObjectInfoOrErr: Send + 'static;
|
||||
type WalkOptions: Send + 'static;
|
||||
type WalkCancellation: Send + 'static;
|
||||
type WalkResultSender: Send + 'static;
|
||||
|
||||
async fn list_objects_v2(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<String>,
|
||||
delimiter: Option<String>,
|
||||
max_keys: i32,
|
||||
fetch_owner: bool,
|
||||
start_after: Option<String>,
|
||||
incl_deleted: bool,
|
||||
) -> Result<Self::ListObjectsV2Info, Self::Error>;
|
||||
|
||||
async fn list_object_versions(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
marker: Option<String>,
|
||||
version_marker: Option<String>,
|
||||
delimiter: Option<String>,
|
||||
max_keys: i32,
|
||||
) -> Result<Self::ListObjectVersionsInfo, Self::Error>;
|
||||
|
||||
async fn walk(
|
||||
self: Arc<Self>,
|
||||
rx: Self::WalkCancellation,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Self::WalkResultSender,
|
||||
opts: Self::WalkOptions,
|
||||
) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ListObjectsInfo<ObjectItem> {
|
||||
pub is_truncated: bool,
|
||||
@@ -458,6 +502,142 @@ mod tests {
|
||||
assert!(!opts.include_free_versions);
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TestListBackend;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TestListError;
|
||||
|
||||
impl fmt::Display for TestListError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("test list error")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for TestListError {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ListOperations for TestListBackend {
|
||||
type Error = TestListError;
|
||||
type ListObjectsV2Info = ListObjectsV2Info<&'static str>;
|
||||
type ListObjectVersionsInfo = ListObjectVersionsInfo<&'static str>;
|
||||
type ObjectInfoOrErr = ObjectInfoOrErr<&'static str, TestListError>;
|
||||
type WalkOptions = WalkOptions<fn(&str) -> bool>;
|
||||
type WalkCancellation = ();
|
||||
type WalkResultSender = ();
|
||||
|
||||
async fn list_objects_v2(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
continuation_token: Option<String>,
|
||||
delimiter: Option<String>,
|
||||
max_keys: i32,
|
||||
fetch_owner: bool,
|
||||
start_after: Option<String>,
|
||||
incl_deleted: bool,
|
||||
) -> Result<Self::ListObjectsV2Info, Self::Error> {
|
||||
assert_eq!(bucket, "bucket");
|
||||
assert_eq!(prefix, "photos/");
|
||||
assert_eq!(continuation_token.as_deref(), Some("token"));
|
||||
assert_eq!(delimiter.as_deref(), Some("/"));
|
||||
assert_eq!(max_keys, 10);
|
||||
assert!(fetch_owner);
|
||||
assert_eq!(start_after.as_deref(), Some("photos/0001.jpg"));
|
||||
assert!(!incl_deleted);
|
||||
|
||||
Ok(ListObjectsV2Info {
|
||||
is_truncated: true,
|
||||
continuation_token,
|
||||
next_continuation_token: Some("next".to_owned()),
|
||||
objects: vec!["photos/0002.jpg"],
|
||||
prefixes: vec!["photos/2024/".to_owned()],
|
||||
})
|
||||
}
|
||||
|
||||
async fn list_object_versions(
|
||||
self: Arc<Self>,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
marker: Option<String>,
|
||||
version_marker: Option<String>,
|
||||
delimiter: Option<String>,
|
||||
max_keys: i32,
|
||||
) -> Result<Self::ListObjectVersionsInfo, Self::Error> {
|
||||
assert_eq!(bucket, "bucket");
|
||||
assert_eq!(prefix, "photos/");
|
||||
assert_eq!(marker.as_deref(), Some("photos/0001.jpg"));
|
||||
assert_eq!(version_marker.as_deref(), Some("version"));
|
||||
assert_eq!(delimiter.as_deref(), Some("/"));
|
||||
assert_eq!(max_keys, 10);
|
||||
|
||||
Ok(ListObjectVersionsInfo {
|
||||
is_truncated: false,
|
||||
next_marker: None,
|
||||
next_version_idmarker: None,
|
||||
objects: vec!["photos/0001.jpg"],
|
||||
prefixes: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn walk(
|
||||
self: Arc<Self>,
|
||||
_rx: Self::WalkCancellation,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
_result: Self::WalkResultSender,
|
||||
opts: Self::WalkOptions,
|
||||
) -> Result<(), Self::Error> {
|
||||
assert_eq!(bucket, "bucket");
|
||||
assert_eq!(prefix, "photos/");
|
||||
assert_eq!(opts.marker.as_deref(), Some("photos/0001.jpg"));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_operations_trait_exposes_generic_list_contract() -> Result<(), TestListError> {
|
||||
let backend = Arc::new(TestListBackend);
|
||||
|
||||
let listed = backend
|
||||
.clone()
|
||||
.list_objects_v2(
|
||||
"bucket",
|
||||
"photos/",
|
||||
Some("token".to_owned()),
|
||||
Some("/".to_owned()),
|
||||
10,
|
||||
true,
|
||||
Some("photos/0001.jpg".to_owned()),
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
assert!(listed.is_truncated);
|
||||
assert_eq!(listed.objects, vec!["photos/0002.jpg"]);
|
||||
assert_eq!(listed.next_continuation_token.as_deref(), Some("next"));
|
||||
|
||||
let versions = backend
|
||||
.clone()
|
||||
.list_object_versions(
|
||||
"bucket",
|
||||
"photos/",
|
||||
Some("photos/0001.jpg".to_owned()),
|
||||
Some("version".to_owned()),
|
||||
Some("/".to_owned()),
|
||||
10,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(versions.objects, vec!["photos/0001.jpg"]);
|
||||
|
||||
let opts = WalkOptions::<fn(&str) -> bool> {
|
||||
marker: Some("photos/0001.jpg".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
backend.walk((), "bucket", "photos/", (), opts).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_list_response_contracts_default_to_empty_collections() {
|
||||
let v1 = ListObjectsInfo::<()>::default();
|
||||
|
||||
@@ -95,7 +95,7 @@ Required `rustfs-storage-api` public re-exports:
|
||||
- `pub use error::{StorageErrorCode, StorageResult};`
|
||||
- `pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};`
|
||||
- `pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};`
|
||||
- `pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ObjectInfoOrErr};`
|
||||
- `pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};`
|
||||
- `pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};`
|
||||
- `pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};`
|
||||
|
||||
|
||||
@@ -5,17 +5,18 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
## Current Context
|
||||
|
||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||
- Branch: `overtrue/arch-walk-options-contracts`
|
||||
- Baseline: `main` at `604379d62fab481b73d1848bfbd045a684031441`
|
||||
after the table catalog row-level conflict hardening merge.
|
||||
- Branch: `overtrue/arch-list-operations-contracts`
|
||||
- Baseline: `main` at `36f7ad6936d03b9593880b5c4958f47b2a039fc9`
|
||||
after the walk options contract merge.
|
||||
- PR type for this branch: `api-extraction`
|
||||
- Runtime behavior changes: no external behavior change expected.
|
||||
- Rust code changes: move `WalkOptions` into `rustfs-storage-api` as a generic
|
||||
filter-container contract, then keep ECStore's existing public name as a
|
||||
`FileInfo` filter alias.
|
||||
- CI/script changes: extend migration guards for the `WalkOptions` public
|
||||
- Rust code changes: move `ListOperations` into `rustfs-storage-api` as a
|
||||
generic operation contract with associated ECStore-bound types, then keep
|
||||
ECStore's existing public name as a fixed associated-type compatibility
|
||||
subtrait.
|
||||
- CI/script changes: extend migration guards for the `ListOperations` public
|
||||
re-export and ECStore local-definition regressions.
|
||||
- Docs changes: record the walk options contract extraction slice.
|
||||
- Docs changes: record the list operations contract extraction slice.
|
||||
|
||||
## Phase 0 Tasks
|
||||
|
||||
@@ -661,6 +662,28 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
checks, migration/layer guards, formatting, diff hygiene, Rust risk scan,
|
||||
full pre-commit, and required three-expert review passed.
|
||||
|
||||
- [x] `API-021` Move list operations contract.
|
||||
- Completed slice: move `ListOperations` from ECStore `store_api/traits.rs`
|
||||
into `rustfs-storage-api` as a generic public operation contract over list
|
||||
response, walk option, cancellation, sender, and error associated types;
|
||||
keep ECStore's old public `ListOperations` name as a fixed associated-type
|
||||
compatibility subtrait.
|
||||
- Acceptance: `rustfs-storage-api` exports `ListOperations`, ECStore no
|
||||
longer defines local list operation method signatures, existing ECStore
|
||||
generic bounds keep the old import path, and migration guards reject
|
||||
dropping the public storage-api re-export or reintroducing local ECStore
|
||||
list method definitions.
|
||||
- Must preserve: list v2 pagination, list-object-versions pagination, walk
|
||||
channel shape, cancellation token usage, ECStore public compatibility
|
||||
bounds, and all ECStore list/walk runtime behavior.
|
||||
- Risk defense: only the trait contract crosses into `rustfs-storage-api`;
|
||||
ECStore keeps the concrete associated type bindings, response aliases,
|
||||
walk option alias, object metadata conversion, storage errors, lifecycle
|
||||
and replication coupling, and implementation bodies.
|
||||
- Verification: focused storage-api tests, ECStore/RustFS/downstream compile
|
||||
checks, migration/layer guards, formatting, diff hygiene, Rust risk scan,
|
||||
full pre-commit, and required three-expert review passed.
|
||||
|
||||
## Phase 8 Background Controller Tasks
|
||||
|
||||
- [x] `BGC-001` Inventory background services.
|
||||
@@ -937,8 +960,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
|
||||
| Expert | Status | Notes |
|
||||
|---|---|---|
|
||||
| Quality/architecture | passed | Generic `WalkOptions` now lives in `rustfs-storage-api`; ECStore still owns the concrete `FileInfo` filter binding and list/walk implementation behavior. |
|
||||
| Migration preservation | passed | Filter optionality, marker/default fields, version sort default, include-free-versions flag, and existing ECStore public import path are preserved through an alias. |
|
||||
| Quality/architecture | passed | Generic `ListOperations` now lives in `rustfs-storage-api`; ECStore still owns the concrete associated type bindings and implementation behavior. |
|
||||
| Migration preservation | passed | List v2, list-object-versions, walk channel/cancellation shape, and existing ECStore generic bound import path are preserved through a compatibility subtrait. |
|
||||
| Testing/verification | passed | Focused storage-api tests, downstream compile checks, migration/layer guards, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` passed. |
|
||||
|
||||
## Verification Notes
|
||||
@@ -954,25 +977,26 @@ Passed before push:
|
||||
- Rust risk scan: no new `unwrap`/`expect`, panic/todo markers, `unsafe`,
|
||||
process-spawning calls, lossy casts, println/eprintln, or relaxed ordering in
|
||||
added Rust lines.
|
||||
- `make pre-commit`: passed; nextest reported 6203 tests passed and 111
|
||||
- `make pre-commit`: passed; nextest reported 6204 tests passed and 111
|
||||
skipped, and doctests passed.
|
||||
|
||||
Notes:
|
||||
|
||||
- This slice follows the object list response contract branch and keeps the old
|
||||
- This slice follows the walk options contract branch and keeps the old
|
||||
aggregate facade, bucket DTO, multipart DTO, bucket operation contract, object
|
||||
helper, range helper, list helper, object precondition, list response, and
|
||||
walk options contract guards active.
|
||||
- The shared walk options container is now owned by `rustfs-storage-api`;
|
||||
ECStore keeps the concrete `FileInfo` filter alias, `ObjectInfo`, storage
|
||||
`Error`, `ObjectOptions`, object metadata adaptation, storage error mapping,
|
||||
readers, lifecycle/replication, rio, filemeta, and implementation behavior.
|
||||
- The slice does not alter object, list, walk, multipart, bucket, delete, or
|
||||
reader runtime behavior.
|
||||
- The shared list operations contract is now owned by `rustfs-storage-api`;
|
||||
ECStore keeps the concrete associated type bindings, response aliases,
|
||||
`WalkOptions` alias, `ObjectInfo`, storage `Error`, `ObjectOptions`, object
|
||||
metadata adaptation, storage error mapping, readers, lifecycle/replication,
|
||||
rio, filemeta, and implementation behavior.
|
||||
- The slice does not alter object, list, walk, multipart, bucket, delete,
|
||||
namespace-lock, or reader runtime behavior.
|
||||
|
||||
## Handoff Notes
|
||||
|
||||
- Walk options contract cleanup is stacked on the object list response contract
|
||||
- List operations contract cleanup is stacked on the walk options contract
|
||||
branch.
|
||||
- After this lands, remaining storage work can continue by extracting larger
|
||||
low-coupling DTO/consumer slices or by narrowing remaining operation-group
|
||||
|
||||
@@ -73,8 +73,8 @@ use rustfs_ecstore::config::com::STORAGE_CLASS_SUB_SYS;
|
||||
use rustfs_ecstore::config::com::{delete_config, read_config, read_config_without_migrate, save_config, save_server_config};
|
||||
use rustfs_ecstore::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
|
||||
use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
|
||||
use rustfs_ecstore::store_api::ListOperations;
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
use rustfs_storage_api::ListOperations as _;
|
||||
use s3s::header::CONTENT_TYPE;
|
||||
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
|
||||
use serde::Serialize;
|
||||
|
||||
@@ -33,10 +33,10 @@ use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
|
||||
use rustfs_credentials::get_global_action_cred;
|
||||
use rustfs_ecstore::{
|
||||
global::get_global_region,
|
||||
store_api::{ListOperations, ObjectIO, ObjectOperations, ObjectOptions},
|
||||
store_api::{ObjectIO, ObjectOperations, ObjectOptions},
|
||||
};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_storage_api::{BucketOperations, bucket::BucketOptions};
|
||||
use rustfs_storage_api::{BucketOperations, ListOperations as _, bucket::BucketOptions};
|
||||
use rustfs_trusted_proxies::{ClientInfo, ValidationMode};
|
||||
use rustfs_utils::{base64_decode_url_safe_no_pad, base64_encode_url_safe_no_pad};
|
||||
use s3s::{Body, S3Request, S3Response, S3Result, dto::StreamingBlob, header::CONTENT_TYPE, s3_error};
|
||||
|
||||
@@ -57,13 +57,14 @@ use rustfs_ecstore::client::object_api_utils::to_s3s_etag;
|
||||
use rustfs_ecstore::error::StorageError;
|
||||
use rustfs_ecstore::notification_sys::get_global_notification_sys;
|
||||
use rustfs_ecstore::store::ECStore;
|
||||
use rustfs_ecstore::store_api::{ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, ObjectInfo};
|
||||
use rustfs_ecstore::store_api::{ListObjectVersionsInfo, ListObjectsV2Info, ObjectInfo};
|
||||
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
|
||||
use rustfs_policy::policy::{
|
||||
action::{Action, S3Action},
|
||||
{BucketPolicy, BucketPolicyArgs, Effect, Validator},
|
||||
};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_storage_api::ListOperations as _;
|
||||
use rustfs_storage_api::{BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions};
|
||||
use rustfs_targets::{
|
||||
EventName,
|
||||
|
||||
@@ -29,13 +29,14 @@ use rustfs_ecstore::{
|
||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
global::GLOBAL_TierConfigMgr,
|
||||
store::ECStore,
|
||||
store_api::{ListOperations, MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
|
||||
store_api::{MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
|
||||
tier::{
|
||||
tier_config::{TierConfig, TierType},
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||
},
|
||||
};
|
||||
use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager};
|
||||
use rustfs_storage_api::ListOperations as _;
|
||||
use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions};
|
||||
use rustfs_utils::http::{SUFFIX_FORCE_DELETE, insert_header};
|
||||
use s3s::{S3Request, dto::*};
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_ecstore::store::ECStore;
|
||||
use rustfs_ecstore::store_api::ListOperations;
|
||||
use rustfs_storage_api::ListOperations as _;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Determines if the key "looks like a prefix" (ends with `/`).
|
||||
|
||||
@@ -199,7 +199,7 @@ require_source_line \
|
||||
"storage-api public object helper contract re-export"
|
||||
require_source_line \
|
||||
"crates/storage-api/src/lib.rs" \
|
||||
"pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ObjectInfoOrErr};" \
|
||||
"pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};" \
|
||||
"storage-api public list response contract re-export"
|
||||
require_source_line \
|
||||
"crates/storage-api/src/lib.rs" \
|
||||
@@ -285,7 +285,7 @@ fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ObjectInfoOrErr)\b|::(?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ObjectInfoOrErr)\b)|pub struct (?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ObjectInfoOrErr)\b' \
|
||||
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ListOperations|ObjectInfoOrErr)\b|::(?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ListOperations|ObjectInfoOrErr)\b)|pub struct (?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ObjectInfoOrErr)\b' \
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true
|
||||
) >"$STORE_API_LIST_RESPONSE_REEXPORTS_FILE"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user