From 3fb4cb3d65a2e037fc2e5ede32bf81c1f15c9fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Thu, 18 Jun 2026 08:48:18 +0800 Subject: [PATCH] refactor: move list operations contract (#3550) --- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 6 +- .../bucket/lifecycle/tier_delete_journal.rs | 3 +- .../lifecycle/tier_free_version_recovery.rs | 4 +- crates/ecstore/src/data_usage.rs | 2 +- crates/ecstore/src/set_disk.rs | 14 +- crates/ecstore/src/sets.rs | 15 +- crates/ecstore/src/store.rs | 14 +- crates/ecstore/src/store_api/traits.rs | 59 +++--- crates/heal/src/heal/storage.rs | 4 +- crates/iam/src/store/object.rs | 3 +- crates/protocols/src/swift/container.rs | 5 +- crates/protocols/src/swift/versioning.rs | 3 +- .../tests/lifecycle_integration_test.rs | 3 +- crates/storage-api/src/lib.rs | 2 +- crates/storage-api/src/object.rs | 180 ++++++++++++++++++ docs/architecture/crate-boundaries.md | 2 +- docs/architecture/migration-progress.md | 62 ++++-- rustfs/src/admin/handlers/config_admin.rs | 2 +- .../src/admin/handlers/object_zip_download.rs | 4 +- rustfs/src/app/bucket_usecase.rs | 3 +- .../src/app/lifecycle_transition_api_test.rs | 3 +- rustfs/src/storage/head_prefix.rs | 2 +- scripts/check_architecture_migration_rules.sh | 4 +- 23 files changed, 313 insertions(+), 86 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index d41797cf2..58884b774 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -37,9 +37,7 @@ use crate::global::GLOBAL_LocalNodeName; use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id}; use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks}; use crate::store::ECStore; -use crate::store_api::{ - GetObjectReader, ListOperations, MultipartOperations, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, -}; +use crate::store_api::{GetObjectReader, MultipartOperations, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete}; use crate::tier::warm_backend::WarmBackendGetOpts; use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded}; use futures::Future; @@ -60,7 +58,7 @@ use rustfs_filemeta::{ VersionPurgeStatusType, get_file_info, is_restored_object_on_disk, }; use rustfs_s3_types::EventName; -use rustfs_storage_api::HTTPRangeSpec; +use rustfs_storage_api::{HTTPRangeSpec, ListOperations as _}; use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold}; use s3s::dto::{ BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest, diff --git a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs index d8ef828cb..06a4a7473 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs @@ -24,7 +24,8 @@ use crate::config::com::{delete_config, read_config, save_config}; use crate::disk::RUSTFS_META_BUCKET; use crate::error::{Error, Result}; use crate::store::ECStore; -use crate::store_api::{ListOperations, ObjectIO, ObjectOperations}; +use crate::store_api::{ObjectIO, ObjectOperations}; +use rustfs_storage_api::ListOperations as _; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle"; diff --git a/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs b/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs index 6b673eba0..67e607191 100644 --- a/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs +++ b/crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs @@ -20,8 +20,8 @@ use tokio_util::sync::CancellationToken; use crate::disk::RUSTFS_META_BUCKET; use crate::error::Result; use crate::store::ECStore; -use crate::store_api::{ListOperations, ObjectInfo, ObjectInfoOrErr, WalkOptions}; -use rustfs_storage_api::{BucketOperations, BucketOptions}; +use crate::store_api::{ObjectInfo, ObjectInfoOrErr, WalkOptions}; +use rustfs_storage_api::{BucketOperations, BucketOptions, ListOperations as _}; pub const DEFAULT_FREE_VERSION_RECOVERY_LIMIT: usize = 1_000; const DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT: usize = 10_000; diff --git a/crates/ecstore/src/data_usage.rs b/crates/ecstore/src/data_usage.rs index ab4250302..503b9eb4c 100644 --- a/crates/ecstore/src/data_usage.rs +++ b/crates/ecstore/src/data_usage.rs @@ -20,7 +20,6 @@ use crate::{ disk::DiskAPI, error::{Error, classify_system_path_failure_reason}, store::ECStore, - store_api::ListOperations, }; pub use local_snapshot::{ DATA_USAGE_DIR, DATA_USAGE_STATE_DIR, LOCAL_USAGE_SNAPSHOT_VERSION, LocalUsageSnapshot, LocalUsageSnapshotMeta, @@ -31,6 +30,7 @@ use rustfs_data_usage::{ BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary, }; use rustfs_io_metrics::record_system_path_failure; +use rustfs_storage_api::ListOperations as _; use rustfs_utils::path::SLASH_SEPARATOR; use std::{ collections::{HashMap, HashSet, hash_map::Entry}, diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index d8180b3ce..11657a479 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -52,8 +52,8 @@ use crate::{ event_notification::{EventArgs, send_event}, global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure}, store_api::{ - DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, ListOperations, MultipartOperations, NamespaceLocking, - ObjectIO, ObjectInfo, ObjectOperations, PutObjReader, + DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, MultipartOperations, NamespaceLocking, ObjectIO, + ObjectInfo, ObjectOperations, PutObjReader, }, store_init::load_format_erasure, }; @@ -2985,7 +2985,15 @@ impl SetDisks { } #[async_trait::async_trait] -impl ListOperations for SetDisks { +impl rustfs_storage_api::ListOperations for SetDisks { + type Error = Error; + type ListObjectsV2Info = ListObjectsV2Info; + type ListObjectVersionsInfo = ListObjectVersionsInfo; + type ObjectInfoOrErr = ObjectInfoOrErr; + type WalkOptions = WalkOptions; + type WalkCancellation = CancellationToken; + type WalkResultSender = Sender; + #[tracing::instrument(skip(self))] async fn list_objects_v2( self: Arc, diff --git a/crates/ecstore/src/sets.rs b/crates/ecstore/src/sets.rs index 4fe678cd0..6399b5093 100644 --- a/crates/ecstore/src/sets.rs +++ b/crates/ecstore/src/sets.rs @@ -28,9 +28,8 @@ use crate::{ global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure}, set_disk::SetDisks, store_api::{ - DeletedObject, GetObjectReader, HealOperations, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, - MultipartOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, - PutObjReader, + DeletedObject, GetObjectReader, HealOperations, ListObjectVersionsInfo, ListObjectsV2Info, MultipartOperations, + NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader, }, store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file}, }; @@ -647,7 +646,15 @@ impl ObjectOperations for Sets { } #[async_trait::async_trait] -impl ListOperations for Sets { +impl rustfs_storage_api::ListOperations for Sets { + type Error = Error; + type ListObjectsV2Info = ListObjectsV2Info; + type ListObjectVersionsInfo = ListObjectVersionsInfo; + type ObjectInfoOrErr = ObjectInfoOrErr; + type WalkOptions = WalkOptions; + type WalkCancellation = CancellationToken; + type WalkResultSender = tokio::sync::mpsc::Sender; + #[tracing::instrument(skip(self))] async fn list_objects_v2( self: Arc, diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index d2207c417..66eee815a 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -63,8 +63,8 @@ use crate::{ rpc::S3PeerSys, sets::Sets, store_api::{ - DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, ListOperations, MultipartOperations, NamespaceLocking, - ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader, + DeletedObject, GetObjectReader, HealOperations, ListObjectsV2Info, MultipartOperations, NamespaceLocking, ObjectInfo, + ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader, }, store_init, }; @@ -504,7 +504,15 @@ impl ObjectOperations for ECStore { } #[async_trait::async_trait] -impl ListOperations for ECStore { +impl rustfs_storage_api::ListOperations for ECStore { + type Error = Error; + type ListObjectsV2Info = ListObjectsV2Info; + type ListObjectVersionsInfo = ListObjectVersionsInfo; + type ObjectInfoOrErr = ObjectInfoOrErr; + type WalkOptions = WalkOptions; + type WalkCancellation = CancellationToken; + type WalkResultSender = tokio::sync::mpsc::Sender; + // @continuation_token marker // @start_after as marker when continuation_token empty // @delimiter default="/", empty when recursive diff --git a/crates/ecstore/src/store_api/traits.rs b/crates/ecstore/src/store_api/traits.rs index 5f4dea67f..1b38458bd 100644 --- a/crates/ecstore/src/store_api/traits.rs +++ b/crates/ecstore/src/store_api/traits.rs @@ -49,39 +49,34 @@ pub trait ObjectOperations: Send + Sync + Debug { } /// Listing and walking operations. -#[async_trait::async_trait] -#[allow(clippy::too_many_arguments)] -pub trait ListOperations: Send + Sync + Debug { - async fn list_objects_v2( - self: Arc, - bucket: &str, - prefix: &str, - continuation_token: Option, - delimiter: Option, - max_keys: i32, - fetch_owner: bool, - start_after: Option, - incl_deleted: bool, - ) -> Result; +pub trait ListOperations: + rustfs_storage_api::ListOperations< + Error = Error, + ListObjectsV2Info = ListObjectsV2Info, + ListObjectVersionsInfo = ListObjectVersionsInfo, + ObjectInfoOrErr = ObjectInfoOrErr, + WalkOptions = WalkOptions, + WalkCancellation = CancellationToken, + WalkResultSender = tokio::sync::mpsc::Sender, + > + Send + + Sync + + Debug +{ +} - async fn list_object_versions( - self: Arc, - bucket: &str, - prefix: &str, - marker: Option, - version_marker: Option, - delimiter: Option, - max_keys: i32, - ) -> Result; - - async fn walk( - self: Arc, - rx: CancellationToken, - bucket: &str, - prefix: &str, - result: tokio::sync::mpsc::Sender, - opts: WalkOptions, - ) -> Result<()>; +impl ListOperations for T where + T: rustfs_storage_api::ListOperations< + Error = Error, + ListObjectsV2Info = ListObjectsV2Info, + ListObjectVersionsInfo = ListObjectVersionsInfo, + ObjectInfoOrErr = ObjectInfoOrErr, + WalkOptions = WalkOptions, + WalkCancellation = CancellationToken, + WalkResultSender = tokio::sync::mpsc::Sender, + > + Send + + Sync + + Debug +{ } /// Multipart upload operations. diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index a0eb8df58..3afd5e360 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -19,10 +19,10 @@ use rustfs_ecstore::{ disk::{DiskStore, endpoint::Endpoint}, error::StorageError, store::ECStore, - store_api::{HealOperations, ListOperations, ObjectIO, ObjectOperations, ObjectOptions}, + store_api::{HealOperations, ObjectIO, ObjectOperations, ObjectOptions}, }; use rustfs_madmin::heal_commands::HealResultItem; -use rustfs_storage_api::{BucketInfo, BucketOperations, DiskSetSelector, StorageAdminApi}; +use rustfs_storage_api::{BucketInfo, BucketOperations, DiskSetSelector, ListOperations as _, StorageAdminApi}; use std::sync::Arc; use tracing::{debug, error, warn}; diff --git a/crates/iam/src/store/object.rs b/crates/iam/src/store/object.rs index 916f0d617..6709b12e9 100644 --- a/crates/iam/src/store/object.rs +++ b/crates/iam/src/store/object.rs @@ -23,7 +23,7 @@ use crate::{ use futures::future::join_all; use rustfs_credentials::get_global_action_cred; use rustfs_ecstore::error::{StorageError, classify_system_path_failure_reason}; -use rustfs_ecstore::store_api::{ListOperations as _, ObjectInfoOrErr, WalkOptions}; +use rustfs_ecstore::store_api::{ObjectInfoOrErr, WalkOptions}; use rustfs_ecstore::{ config::{ RUSTFS_CONFIG_PREFIX, @@ -35,6 +35,7 @@ use rustfs_ecstore::{ use rustfs_io_metrics::record_system_path_failure; use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc}; use rustfs_storage_api::HTTPPreconditions; +use rustfs_storage_api::ListOperations as _; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use serde::{Serialize, de::DeserializeOwned}; use std::sync::{LazyLock, Mutex}; diff --git a/crates/protocols/src/swift/container.rs b/crates/protocols/src/swift/container.rs index f5ce62035..9af747b8f 100644 --- a/crates/protocols/src/swift/container.rs +++ b/crates/protocols/src/swift/container.rs @@ -21,8 +21,9 @@ use super::types::Container; use super::{SwiftError, SwiftResult}; use rustfs_credentials::Credentials; use rustfs_ecstore::resolve_object_store_handle; -use rustfs_ecstore::store_api::ListOperations; -use rustfs_storage_api::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions}; +use rustfs_storage_api::{ + BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListOperations as _, MakeBucketOptions, +}; use s3s::dto::{Tag, Tagging}; use sha2::{Digest, Sha256}; use tracing::{debug, error}; diff --git a/crates/protocols/src/swift/versioning.rs b/crates/protocols/src/swift/versioning.rs index ec4df951a..77c56585d 100644 --- a/crates/protocols/src/swift/versioning.rs +++ b/crates/protocols/src/swift/versioning.rs @@ -57,7 +57,8 @@ use super::object::{ObjectKeyMapper, head_object}; use super::{SwiftError, SwiftResult}; use rustfs_credentials::Credentials; use rustfs_ecstore::resolve_object_store_handle; -use rustfs_ecstore::store_api::{ListOperations, ObjectOperations, ObjectOptions}; +use rustfs_ecstore::store_api::{ObjectOperations, ObjectOptions}; +use rustfs_storage_api::ListOperations as _; use std::time::{SystemTime, UNIX_EPOCH}; use tracing::{debug, error}; diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index 34e509ca0..3c23781b0 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -28,7 +28,7 @@ use rustfs_ecstore::{ global::GLOBAL_TierConfigMgr, pools::path2_bucket_object_with_base_path, store::ECStore, - store_api::{ListOperations, MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}, + store_api::{MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}, tier::{ tier_config::{TierConfig, TierMinIO, TierType}, warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options}, @@ -38,6 +38,7 @@ use rustfs_filemeta::FileMeta; use rustfs_scanner::scanner::init_data_scanner; use rustfs_scanner::scanner_folder::ScannerItem; use rustfs_scanner::scanner_io::ScannerIODisk; +use rustfs_storage_api::ListOperations as _; use rustfs_storage_api::{BucketOperations, MakeBucketOptions}; use rustfs_utils::path::path_join_buf; use s3s::dto::RestoreRequest; diff --git a/crates/storage-api/src/lib.rs b/crates/storage-api/src/lib.rs index 804ca660f..0f732796d 100644 --- a/crates/storage-api/src/lib.rs +++ b/crates/storage-api/src/lib.rs @@ -25,6 +25,6 @@ pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOption pub use error::{StorageErrorCode, StorageResult}; pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo}; pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions}; -pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ObjectInfoOrErr}; +pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr}; pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState}; pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder}; diff --git a/crates/storage-api/src/object.rs b/crates/storage-api/src/object.rs index 4519b0142..58a73247e 100644 --- a/crates/storage-api/src/object.rs +++ b/crates/storage-api/src/object.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::fmt; +use std::sync::Arc; use time::OffsetDateTime; use uuid::Uuid; @@ -179,6 +180,49 @@ impl Default for WalkOptions { } } +#[async_trait::async_trait] +#[allow(clippy::too_many_arguments)] +pub trait ListOperations: Send + Sync + fmt::Debug { + type Error: std::error::Error + Send + Sync + 'static; + type ListObjectsV2Info: Send + 'static; + type ListObjectVersionsInfo: Send + 'static; + type ObjectInfoOrErr: Send + 'static; + type WalkOptions: Send + 'static; + type WalkCancellation: Send + 'static; + type WalkResultSender: Send + 'static; + + async fn list_objects_v2( + self: Arc, + bucket: &str, + prefix: &str, + continuation_token: Option, + delimiter: Option, + max_keys: i32, + fetch_owner: bool, + start_after: Option, + incl_deleted: bool, + ) -> Result; + + async fn list_object_versions( + self: Arc, + bucket: &str, + prefix: &str, + marker: Option, + version_marker: Option, + delimiter: Option, + max_keys: i32, + ) -> Result; + + async fn walk( + self: Arc, + rx: Self::WalkCancellation, + bucket: &str, + prefix: &str, + result: Self::WalkResultSender, + opts: Self::WalkOptions, + ) -> Result<(), Self::Error>; +} + #[derive(Debug, Default)] pub struct ListObjectsInfo { pub is_truncated: bool, @@ -458,6 +502,142 @@ mod tests { assert!(!opts.include_free_versions); } + #[derive(Debug)] + struct TestListBackend; + + #[derive(Debug)] + struct TestListError; + + impl fmt::Display for TestListError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("test list error") + } + } + + impl std::error::Error for TestListError {} + + #[async_trait::async_trait] + impl ListOperations for TestListBackend { + type Error = TestListError; + type ListObjectsV2Info = ListObjectsV2Info<&'static str>; + type ListObjectVersionsInfo = ListObjectVersionsInfo<&'static str>; + type ObjectInfoOrErr = ObjectInfoOrErr<&'static str, TestListError>; + type WalkOptions = WalkOptions bool>; + type WalkCancellation = (); + type WalkResultSender = (); + + async fn list_objects_v2( + self: Arc, + bucket: &str, + prefix: &str, + continuation_token: Option, + delimiter: Option, + max_keys: i32, + fetch_owner: bool, + start_after: Option, + incl_deleted: bool, + ) -> Result { + assert_eq!(bucket, "bucket"); + assert_eq!(prefix, "photos/"); + assert_eq!(continuation_token.as_deref(), Some("token")); + assert_eq!(delimiter.as_deref(), Some("/")); + assert_eq!(max_keys, 10); + assert!(fetch_owner); + assert_eq!(start_after.as_deref(), Some("photos/0001.jpg")); + assert!(!incl_deleted); + + Ok(ListObjectsV2Info { + is_truncated: true, + continuation_token, + next_continuation_token: Some("next".to_owned()), + objects: vec!["photos/0002.jpg"], + prefixes: vec!["photos/2024/".to_owned()], + }) + } + + async fn list_object_versions( + self: Arc, + bucket: &str, + prefix: &str, + marker: Option, + version_marker: Option, + delimiter: Option, + max_keys: i32, + ) -> Result { + assert_eq!(bucket, "bucket"); + assert_eq!(prefix, "photos/"); + assert_eq!(marker.as_deref(), Some("photos/0001.jpg")); + assert_eq!(version_marker.as_deref(), Some("version")); + assert_eq!(delimiter.as_deref(), Some("/")); + assert_eq!(max_keys, 10); + + Ok(ListObjectVersionsInfo { + is_truncated: false, + next_marker: None, + next_version_idmarker: None, + objects: vec!["photos/0001.jpg"], + prefixes: Vec::new(), + }) + } + + async fn walk( + self: Arc, + _rx: Self::WalkCancellation, + bucket: &str, + prefix: &str, + _result: Self::WalkResultSender, + opts: Self::WalkOptions, + ) -> Result<(), Self::Error> { + assert_eq!(bucket, "bucket"); + assert_eq!(prefix, "photos/"); + assert_eq!(opts.marker.as_deref(), Some("photos/0001.jpg")); + Ok(()) + } + } + + #[tokio::test] + async fn list_operations_trait_exposes_generic_list_contract() -> Result<(), TestListError> { + let backend = Arc::new(TestListBackend); + + let listed = backend + .clone() + .list_objects_v2( + "bucket", + "photos/", + Some("token".to_owned()), + Some("/".to_owned()), + 10, + true, + Some("photos/0001.jpg".to_owned()), + false, + ) + .await?; + assert!(listed.is_truncated); + assert_eq!(listed.objects, vec!["photos/0002.jpg"]); + assert_eq!(listed.next_continuation_token.as_deref(), Some("next")); + + let versions = backend + .clone() + .list_object_versions( + "bucket", + "photos/", + Some("photos/0001.jpg".to_owned()), + Some("version".to_owned()), + Some("/".to_owned()), + 10, + ) + .await?; + assert_eq!(versions.objects, vec!["photos/0001.jpg"]); + + let opts = WalkOptions:: bool> { + marker: Some("photos/0001.jpg".to_owned()), + ..Default::default() + }; + backend.walk((), "bucket", "photos/", (), opts).await?; + + Ok(()) + } + #[test] fn object_list_response_contracts_default_to_empty_collections() { let v1 = ListObjectsInfo::<()>::default(); diff --git a/docs/architecture/crate-boundaries.md b/docs/architecture/crate-boundaries.md index 44718d5ea..5a3271aee 100644 --- a/docs/architecture/crate-boundaries.md +++ b/docs/architecture/crate-boundaries.md @@ -95,7 +95,7 @@ Required `rustfs-storage-api` public re-exports: - `pub use error::{StorageErrorCode, StorageResult};` - `pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};` - `pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};` -- `pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ObjectInfoOrErr};` +- `pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};` - `pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};` - `pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};` diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index 39c9b4ba6..c9b7e15f7 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,17 +5,18 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-walk-options-contracts` -- Baseline: `main` at `604379d62fab481b73d1848bfbd045a684031441` - after the table catalog row-level conflict hardening merge. +- Branch: `overtrue/arch-list-operations-contracts` +- Baseline: `main` at `36f7ad6936d03b9593880b5c4958f47b2a039fc9` + after the walk options contract merge. - PR type for this branch: `api-extraction` - Runtime behavior changes: no external behavior change expected. -- Rust code changes: move `WalkOptions` into `rustfs-storage-api` as a generic - filter-container contract, then keep ECStore's existing public name as a - `FileInfo` filter alias. -- CI/script changes: extend migration guards for the `WalkOptions` public +- Rust code changes: move `ListOperations` into `rustfs-storage-api` as a + generic operation contract with associated ECStore-bound types, then keep + ECStore's existing public name as a fixed associated-type compatibility + subtrait. +- CI/script changes: extend migration guards for the `ListOperations` public re-export and ECStore local-definition regressions. -- Docs changes: record the walk options contract extraction slice. +- Docs changes: record the list operations contract extraction slice. ## Phase 0 Tasks @@ -661,6 +662,28 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block checks, migration/layer guards, formatting, diff hygiene, Rust risk scan, full pre-commit, and required three-expert review passed. +- [x] `API-021` Move list operations contract. + - Completed slice: move `ListOperations` from ECStore `store_api/traits.rs` + into `rustfs-storage-api` as a generic public operation contract over list + response, walk option, cancellation, sender, and error associated types; + keep ECStore's old public `ListOperations` name as a fixed associated-type + compatibility subtrait. + - Acceptance: `rustfs-storage-api` exports `ListOperations`, ECStore no + longer defines local list operation method signatures, existing ECStore + generic bounds keep the old import path, and migration guards reject + dropping the public storage-api re-export or reintroducing local ECStore + list method definitions. + - Must preserve: list v2 pagination, list-object-versions pagination, walk + channel shape, cancellation token usage, ECStore public compatibility + bounds, and all ECStore list/walk runtime behavior. + - Risk defense: only the trait contract crosses into `rustfs-storage-api`; + ECStore keeps the concrete associated type bindings, response aliases, + walk option alias, object metadata conversion, storage errors, lifecycle + and replication coupling, and implementation bodies. + - Verification: focused storage-api tests, ECStore/RustFS/downstream compile + checks, migration/layer guards, formatting, diff hygiene, Rust risk scan, + full pre-commit, and required three-expert review passed. + ## Phase 8 Background Controller Tasks - [x] `BGC-001` Inventory background services. @@ -937,8 +960,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | Generic `WalkOptions` now lives in `rustfs-storage-api`; ECStore still owns the concrete `FileInfo` filter binding and list/walk implementation behavior. | -| Migration preservation | passed | Filter optionality, marker/default fields, version sort default, include-free-versions flag, and existing ECStore public import path are preserved through an alias. | +| Quality/architecture | passed | Generic `ListOperations` now lives in `rustfs-storage-api`; ECStore still owns the concrete associated type bindings and implementation behavior. | +| Migration preservation | passed | List v2, list-object-versions, walk channel/cancellation shape, and existing ECStore generic bound import path are preserved through a compatibility subtrait. | | Testing/verification | passed | Focused storage-api tests, downstream compile checks, migration/layer guards, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` passed. | ## Verification Notes @@ -954,25 +977,26 @@ Passed before push: - Rust risk scan: no new `unwrap`/`expect`, panic/todo markers, `unsafe`, process-spawning calls, lossy casts, println/eprintln, or relaxed ordering in added Rust lines. -- `make pre-commit`: passed; nextest reported 6203 tests passed and 111 +- `make pre-commit`: passed; nextest reported 6204 tests passed and 111 skipped, and doctests passed. Notes: -- This slice follows the object list response contract branch and keeps the old +- This slice follows the walk options contract branch and keeps the old aggregate facade, bucket DTO, multipart DTO, bucket operation contract, object helper, range helper, list helper, object precondition, list response, and walk options contract guards active. -- The shared walk options container is now owned by `rustfs-storage-api`; - ECStore keeps the concrete `FileInfo` filter alias, `ObjectInfo`, storage - `Error`, `ObjectOptions`, object metadata adaptation, storage error mapping, - readers, lifecycle/replication, rio, filemeta, and implementation behavior. -- The slice does not alter object, list, walk, multipart, bucket, delete, or - reader runtime behavior. +- The shared list operations contract is now owned by `rustfs-storage-api`; + ECStore keeps the concrete associated type bindings, response aliases, + `WalkOptions` alias, `ObjectInfo`, storage `Error`, `ObjectOptions`, object + metadata adaptation, storage error mapping, readers, lifecycle/replication, + rio, filemeta, and implementation behavior. +- The slice does not alter object, list, walk, multipart, bucket, delete, + namespace-lock, or reader runtime behavior. ## Handoff Notes -- Walk options contract cleanup is stacked on the object list response contract +- List operations contract cleanup is stacked on the walk options contract branch. - After this lands, remaining storage work can continue by extracting larger low-coupling DTO/consumer slices or by narrowing remaining operation-group diff --git a/rustfs/src/admin/handlers/config_admin.rs b/rustfs/src/admin/handlers/config_admin.rs index acae5ed4c..a66f15c80 100644 --- a/rustfs/src/admin/handlers/config_admin.rs +++ b/rustfs/src/admin/handlers/config_admin.rs @@ -73,8 +73,8 @@ use rustfs_ecstore::config::com::STORAGE_CLASS_SUB_SYS; use rustfs_ecstore::config::com::{delete_config, read_config, read_config_without_migrate, save_config, save_server_config}; use rustfs_ecstore::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV}; use rustfs_ecstore::disk::RUSTFS_META_BUCKET; -use rustfs_ecstore::store_api::ListOperations; use rustfs_policy::policy::action::{Action, AdminAction}; +use rustfs_storage_api::ListOperations as _; use s3s::header::CONTENT_TYPE; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use serde::Serialize; diff --git a/rustfs/src/admin/handlers/object_zip_download.rs b/rustfs/src/admin/handlers/object_zip_download.rs index dcf1c7db9..96d84267f 100644 --- a/rustfs/src/admin/handlers/object_zip_download.rs +++ b/rustfs/src/admin/handlers/object_zip_download.rs @@ -33,10 +33,10 @@ use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_credentials::get_global_action_cred; use rustfs_ecstore::{ global::get_global_region, - store_api::{ListOperations, ObjectIO, ObjectOperations, ObjectOptions}, + store_api::{ObjectIO, ObjectOperations, ObjectOptions}, }; use rustfs_policy::policy::action::{Action, S3Action}; -use rustfs_storage_api::{BucketOperations, bucket::BucketOptions}; +use rustfs_storage_api::{BucketOperations, ListOperations as _, bucket::BucketOptions}; use rustfs_trusted_proxies::{ClientInfo, ValidationMode}; use rustfs_utils::{base64_decode_url_safe_no_pad, base64_encode_url_safe_no_pad}; use s3s::{Body, S3Request, S3Response, S3Result, dto::StreamingBlob, header::CONTENT_TYPE, s3_error}; diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 381c69d68..e6f9c3b2e 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -57,13 +57,14 @@ use rustfs_ecstore::client::object_api_utils::to_s3s_etag; use rustfs_ecstore::error::StorageError; use rustfs_ecstore::notification_sys::get_global_notification_sys; use rustfs_ecstore::store::ECStore; -use rustfs_ecstore::store_api::{ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, ObjectInfo}; +use rustfs_ecstore::store_api::{ListObjectVersionsInfo, ListObjectsV2Info, ObjectInfo}; use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta}; use rustfs_policy::policy::{ action::{Action, S3Action}, {BucketPolicy, BucketPolicyArgs, Effect, Validator}, }; use rustfs_s3_ops::S3Operation; +use rustfs_storage_api::ListOperations as _; use rustfs_storage_api::{BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions}; use rustfs_targets::{ EventName, diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index bae165eec..b8ed15625 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -29,13 +29,14 @@ use rustfs_ecstore::{ endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, global::GLOBAL_TierConfigMgr, store::ECStore, - store_api::{ListOperations, MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}, + store_api::{MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}, tier::{ tier_config::{TierConfig, TierType}, warm_backend::{WarmBackend, WarmBackendGetOpts}, }, }; use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager}; +use rustfs_storage_api::ListOperations as _; use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions}; use rustfs_utils::http::{SUFFIX_FORCE_DELETE, insert_header}; use s3s::{S3Request, dto::*}; diff --git a/rustfs/src/storage/head_prefix.rs b/rustfs/src/storage/head_prefix.rs index 651c0dc36..05bbf4293 100644 --- a/rustfs/src/storage/head_prefix.rs +++ b/rustfs/src/storage/head_prefix.rs @@ -13,7 +13,7 @@ // limitations under the License. use rustfs_ecstore::store::ECStore; -use rustfs_ecstore::store_api::ListOperations; +use rustfs_storage_api::ListOperations as _; use std::sync::Arc; /// Determines if the key "looks like a prefix" (ends with `/`). diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 002a4fefa..277b7125a 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -199,7 +199,7 @@ require_source_line \ "storage-api public object helper contract re-export" require_source_line \ "crates/storage-api/src/lib.rs" \ - "pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ObjectInfoOrErr};" \ + "pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};" \ "storage-api public list response contract re-export" require_source_line \ "crates/storage-api/src/lib.rs" \ @@ -285,7 +285,7 @@ fi ( cd "$ROOT_DIR" - rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ObjectInfoOrErr)\b|::(?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ObjectInfoOrErr)\b)|pub struct (?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ObjectInfoOrErr)\b' \ + rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ListOperations|ObjectInfoOrErr)\b|::(?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ListOperations|ObjectInfoOrErr)\b)|pub struct (?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ObjectInfoOrErr)\b' \ crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true ) >"$STORE_API_LIST_RESPONSE_REEXPORTS_FILE"