diff --git a/Cargo.lock b/Cargo.lock index 9209ef19b..e50d073da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9503,6 +9503,7 @@ dependencies = [ "rustfs-io-metrics", "rustfs-madmin", "rustfs-policy", + "rustfs-storage-api", "rustfs-utils", "serde", "serde_json", diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index a09925f54..9b7c70fd4 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -4145,7 +4145,7 @@ mod tests { &bucket, object, &upload.upload_id, - vec![crate::store_api::CompletePart { + vec![rustfs_storage_api::CompletePart { part_num: 1, etag: second_part.etag.clone(), checksum_crc32: None, diff --git a/crates/ecstore/src/data_movement.rs b/crates/ecstore/src/data_movement.rs index 3c1cb8b91..83b50e8a6 100644 --- a/crates/ecstore/src/data_movement.rs +++ b/crates/ecstore/src/data_movement.rs @@ -15,10 +15,11 @@ use crate::error::{Error, Result, is_err_data_movement_overwrite, is_err_object_not_found, is_err_version_not_found}; use crate::store::ECStore; use crate::store_api::{ - CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, PutObjReader, + GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, PutObjReader, }; use bytes::Bytes; use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex}; +use rustfs_storage_api::CompletePart; use rustfs_utils::path::encode_dir_object; use std::io::Cursor; use std::pin::Pin; diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 2ad5c4bce..991ac4d29 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -52,8 +52,8 @@ use crate::{ event_notification::{EventArgs, send_event}, global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure}, store_api::{ - CompletePart, DeletedObject, GetObjectReader, HTTPRangeSpec, HealOperations, ListObjectsV2Info, ListOperations, - MultipartOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, PutObjReader, + DeletedObject, GetObjectReader, HTTPRangeSpec, HealOperations, ListObjectsV2Info, ListOperations, MultipartOperations, + NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, PutObjReader, }, store_init::load_format_erasure, }; @@ -86,8 +86,8 @@ use rustfs_object_capacity::capacity_scope::{ }; use rustfs_s3_types::EventName; use rustfs_storage_api::{ - BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo, MakeBucketOptions, - MultipartInfo, MultipartUploadResult, PartInfo, + BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo, + MakeBucketOptions, MultipartInfo, MultipartUploadResult, PartInfo, }; use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING; use rustfs_utils::http::headers::AMZ_STORAGE_CLASS; @@ -4974,7 +4974,7 @@ mod tests { use crate::disk::health_state::RuntimeDriveHealthState; use crate::endpoints::SetupType; use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type}; - use crate::store_api::{CompletePart, ObjectInfo}; + use crate::store_api::ObjectInfo; use crate::store_init::save_format_file; use crate::store_list_objects::ListPathOptions; use rustfs_filemeta::ErasureInfo; @@ -4982,6 +4982,7 @@ mod tests { use rustfs_filemeta::ReplicationState; use rustfs_lock::client::local::LocalClient; use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats}; + use rustfs_storage_api::CompletePart; use serial_test::serial; use std::collections::HashMap; use tempfile::TempDir; @@ -6672,7 +6673,7 @@ mod tests { ..Default::default() }; let opts = ObjectOptions { - object_lock_retention: Some(crate::store_api::ObjectLockRetentionOptions { + object_lock_retention: Some(rustfs_storage_api::ObjectLockRetentionOptions { mode: Some(s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string()), retain_until: Some(requested_until), bypass_governance: true, @@ -6707,7 +6708,7 @@ mod tests { ..Default::default() }; let opts = ObjectOptions { - object_lock_retention: Some(crate::store_api::ObjectLockRetentionOptions { + object_lock_retention: Some(rustfs_storage_api::ObjectLockRetentionOptions { mode: Some(s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string()), retain_until: Some(requested_until), bypass_governance: true, diff --git a/crates/ecstore/src/sets.rs b/crates/ecstore/src/sets.rs index e5a79844a..3afae538d 100644 --- a/crates/ecstore/src/sets.rs +++ b/crates/ecstore/src/sets.rs @@ -28,9 +28,9 @@ use crate::{ global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure}, set_disk::SetDisks, store_api::{ - CompletePart, DeletedObject, GetObjectReader, HTTPRangeSpec, HealOperations, ListObjectVersionsInfo, ListObjectsV2Info, - ListOperations, MultipartOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, - ObjectToDelete, PutObjReader, + DeletedObject, GetObjectReader, HTTPRangeSpec, HealOperations, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, + MultipartOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, + PutObjReader, }, store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file}, }; @@ -48,6 +48,7 @@ use rustfs_filemeta::FileInfo; use rustfs_lock::NamespaceLockWrapper; use rustfs_lock::client::LockClient; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem}; +use rustfs_storage_api::CompletePart; use rustfs_storage_api::{ BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, PartInfo, diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index 5bf6e2c03..43801c51b 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -63,8 +63,8 @@ use crate::{ rpc::S3PeerSys, sets::Sets, store_api::{ - CompletePart, DeletedObject, GetObjectReader, HTTPRangeSpec, HealOperations, ListObjectsV2Info, ListOperations, - MultipartOperations, NamespaceLocking, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader, + DeletedObject, GetObjectReader, HTTPRangeSpec, HealOperations, ListObjectsV2Info, ListOperations, MultipartOperations, + NamespaceLocking, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PutObjReader, }, store_init, }; @@ -79,8 +79,8 @@ use rustfs_filemeta::FileInfo; use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper}; use rustfs_madmin::heal_commands::HealResultItem; use rustfs_storage_api::{ - BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo, MakeBucketOptions, - MultipartInfo, MultipartUploadResult, PartInfo, + BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, ListMultipartsInfo, ListPartsInfo, + MakeBucketOptions, MultipartInfo, MultipartUploadResult, PartInfo, }; use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf}; use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration}; diff --git a/crates/ecstore/src/store_api/traits.rs b/crates/ecstore/src/store_api/traits.rs index e5eb2de23..5f4dea67f 100644 --- a/crates/ecstore/src/store_api/traits.rs +++ b/crates/ecstore/src/store_api/traits.rs @@ -1,5 +1,5 @@ use super::*; -use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo}; +use rustfs_storage_api::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo}; #[async_trait::async_trait] pub trait ObjectIO: Send + Sync + Debug + 'static { diff --git a/crates/ecstore/src/store_api/types.rs b/crates/ecstore/src/store_api/types.rs index b65360e65..b2731fc8c 100644 --- a/crates/ecstore/src/store_api/types.rs +++ b/crates/ecstore/src/store_api/types.rs @@ -1,29 +1,5 @@ use super::*; - -#[derive(Debug, Default, Clone)] -pub struct HTTPPreconditions { - pub if_match: Option, - pub if_none_match: Option, - pub if_modified_since: Option, - pub if_unmodified_since: Option, -} - -impl HTTPPreconditions { - pub(crate) fn if_match_value(&self) -> Option<&str> { - non_empty_condition_value(self.if_match.as_deref()) - } - - pub(crate) fn if_none_match_value(&self) -> Option<&str> { - non_empty_condition_value(self.if_none_match.as_deref()) - } -} - -#[derive(Debug, Default, Clone)] -pub struct ObjectLockRetentionOptions { - pub mode: Option, - pub retain_until: Option, - pub bypass_governance: bool, -} +use rustfs_storage_api::{HTTPPreconditions, ObjectLockRetentionOptions}; #[derive(Debug, Default, Clone)] pub struct ObjectOptions { @@ -180,10 +156,6 @@ impl ObjectOptions { } } -fn non_empty_condition_value(value: Option<&str>) -> Option<&str> { - value.map(str::trim).filter(|value| !value.is_empty()) -} - fn is_etag_equal(etag1: &str, etag2: &str) -> bool { let e1 = etag1.trim_matches('"'); let e2 = etag2.trim_matches('"'); @@ -200,32 +172,6 @@ fn is_modified_since(mod_time: &OffsetDateTime, given_time: &OffsetDateTime) -> mod_secs > given_secs } -#[derive(Debug, Clone, Default)] -pub struct CompletePart { - pub part_num: usize, - pub etag: Option, - // pub size: Option, - pub checksum_crc32: Option, - pub checksum_crc32c: Option, - pub checksum_sha1: Option, - pub checksum_sha256: Option, - pub checksum_crc64nvme: Option, -} - -impl From for CompletePart { - fn from(value: s3s::dto::CompletedPart) -> Self { - Self { - part_num: value.part_number.unwrap_or_default() as usize, - etag: value.e_tag.map(|v| v.value().to_owned()), - checksum_crc32: value.checksum_crc32, - checksum_crc32c: value.checksum_crc32c, - checksum_sha1: value.checksum_sha1, - checksum_sha256: value.checksum_sha256, - checksum_crc64nvme: value.checksum_crc64nvme, - } - } -} - #[derive(Debug, Default)] pub struct ObjectInfo { pub bucket: String, diff --git a/crates/iam/Cargo.toml b/crates/iam/Cargo.toml index 5a9831448..e81714359 100644 --- a/crates/iam/Cargo.toml +++ b/crates/iam/Cargo.toml @@ -35,6 +35,7 @@ tokio.workspace = true time = { workspace = true, features = ["serde-human-readable"] } serde = { workspace = true, features = ["derive", "rc"] } rustfs-ecstore = { workspace = true } +rustfs-storage-api = { workspace = true } rustfs-policy.workspace = true serde_json.workspace = true async-trait.workspace = true diff --git a/crates/iam/src/store/object.rs b/crates/iam/src/store/object.rs index 4a367710f..916f0d617 100644 --- a/crates/iam/src/store/object.rs +++ b/crates/iam/src/store/object.rs @@ -30,10 +30,11 @@ use rustfs_ecstore::{ com::{delete_config, read_config_no_lock, read_config_with_metadata, save_config, save_config_with_opts}, }, store::ECStore, - store_api::{HTTPPreconditions, ObjectInfo, ObjectOptions}, + store_api::{ObjectInfo, ObjectOptions}, }; use rustfs_io_metrics::record_system_path_failure; use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc}; +use rustfs_storage_api::HTTPPreconditions; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use serde::{Serialize, de::DeserializeOwned}; use std::sync::{LazyLock, Mutex}; diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index c920e8e1f..34e509ca0 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -1020,7 +1020,7 @@ mod serial_tests { multipart_bucket.as_str(), multipart_object, &upload.upload_id, - vec![rustfs_ecstore::store_api::CompletePart { + vec![rustfs_storage_api::CompletePart { part_num: 1, etag: part.etag.clone(), ..Default::default() @@ -1165,12 +1165,12 @@ mod serial_tests { object_name, &upload.upload_id, vec![ - rustfs_ecstore::store_api::CompletePart { + rustfs_storage_api::CompletePart { part_num: 1, etag: uploaded_part1.etag.clone(), ..Default::default() }, - rustfs_ecstore::store_api::CompletePart { + rustfs_storage_api::CompletePart { part_num: 2, etag: uploaded_part2.etag.clone(), ..Default::default() diff --git a/crates/storage-api/src/lib.rs b/crates/storage-api/src/lib.rs index 313e12b5b..bbbd5af24 100644 --- a/crates/storage-api/src/lib.rs +++ b/crates/storage-api/src/lib.rs @@ -18,8 +18,10 @@ pub mod admin; pub mod bucket; pub mod error; pub mod multipart; +pub mod object; pub use admin::{DiskSetSelector, StorageAdminApi}; pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp}; pub use error::{StorageErrorCode, StorageResult}; -pub use multipart::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo}; +pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo}; +pub use object::{HTTPPreconditions, ObjectLockRetentionOptions}; diff --git a/crates/storage-api/src/multipart.rs b/crates/storage-api/src/multipart.rs index 2264bfb1c..ccf4d6468 100644 --- a/crates/storage-api/src/multipart.rs +++ b/crates/storage-api/src/multipart.rs @@ -69,3 +69,32 @@ pub struct ListPartsInfo { pub checksum_algorithm: String, pub checksum_type: String, } + +#[derive(Debug, Clone, Default)] +pub struct CompletePart { + pub part_num: usize, + pub etag: Option, + pub checksum_crc32: Option, + pub checksum_crc32c: Option, + pub checksum_sha1: Option, + pub checksum_sha256: Option, + pub checksum_crc64nvme: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn complete_part_defaults_preserve_empty_checksums() { + let part = CompletePart::default(); + + assert_eq!(part.part_num, 0); + assert!(part.etag.is_none()); + assert!(part.checksum_crc32.is_none()); + assert!(part.checksum_crc32c.is_none()); + assert!(part.checksum_sha1.is_none()); + assert!(part.checksum_sha256.is_none()); + assert!(part.checksum_crc64nvme.is_none()); + } +} diff --git a/crates/storage-api/src/object.rs b/crates/storage-api/src/object.rs new file mode 100644 index 000000000..2c7141a02 --- /dev/null +++ b/crates/storage-api/src/object.rs @@ -0,0 +1,70 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use time::OffsetDateTime; + +#[derive(Debug, Default, Clone)] +pub struct HTTPPreconditions { + pub if_match: Option, + pub if_none_match: Option, + pub if_modified_since: Option, + pub if_unmodified_since: Option, +} + +impl HTTPPreconditions { + pub fn if_match_value(&self) -> Option<&str> { + non_empty_condition_value(self.if_match.as_deref()) + } + + pub fn if_none_match_value(&self) -> Option<&str> { + non_empty_condition_value(self.if_none_match.as_deref()) + } +} + +#[derive(Debug, Default, Clone)] +pub struct ObjectLockRetentionOptions { + pub mode: Option, + pub retain_until: Option, + pub bypass_governance: bool, +} + +fn non_empty_condition_value(value: Option<&str>) -> Option<&str> { + value.map(str::trim).filter(|value| !value.is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn http_preconditions_ignore_empty_etag_headers() { + let opts = HTTPPreconditions { + if_match: Some(" ".to_owned()), + if_none_match: Some(" * ".to_owned()), + ..Default::default() + }; + + assert_eq!(opts.if_match_value(), None); + assert_eq!(opts.if_none_match_value(), Some("*")); + } + + #[test] + fn object_lock_retention_defaults_preserve_false_bypass() { + let opts = ObjectLockRetentionOptions::default(); + + assert!(opts.mode.is_none()); + assert!(opts.retain_until.is_none()); + assert!(!opts.bypass_governance); + } +} diff --git a/docs/architecture/crate-boundaries.md b/docs/architecture/crate-boundaries.md index b81afc036..c63717bc3 100644 --- a/docs/architecture/crate-boundaries.md +++ b/docs/architecture/crate-boundaries.md @@ -93,7 +93,8 @@ Required `rustfs-storage-api` public re-exports: - `pub use admin::{DiskSetSelector, StorageAdminApi};` - `pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};` - `pub use error::{StorageErrorCode, StorageResult};` -- `pub use multipart::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};` +- `pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};` +- `pub use object::{HTTPPreconditions, ObjectLockRetentionOptions};` ECStore must keep compile-time coverage for both `StorageAdminApi` and the separate `NamespaceLocking` operation group. diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index 72a8bcef1..64a8b819c 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,16 +5,17 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-bucket-operations-contract` -- Baseline: `origin/main` at `1830911973f882e30a1de18a5ff7d2ab669d67dc` +- Branch: `overtrue/arch-object-option-contracts` +- Baseline: `origin/main` at `8d24d9133b31c0ca0de0fcd16ad06cd2fdc6f7ae` - PR type for this branch: `api-extraction` - Runtime behavior changes: no external behavior change expected. -- Rust code changes: move the `BucketOperations` contract from ECStore - `store_api` into `rustfs-storage-api` and migrate in-repo consumers to the - shared contract path. -- CI/script changes: extend migration guards for the bucket contract public - re-export and reject restoring the old ECStore-owned `BucketOperations` path. -- Docs changes: record the bucket operation contract extraction slice. +- Rust code changes: move the pure `CompletePart`, `HTTPPreconditions`, and + `ObjectLockRetentionOptions` helper contracts from ECStore `store_api` into + `rustfs-storage-api`, then migrate in-repo consumers to the shared contract + path. +- CI/script changes: extend migration guards for multipart/object helper public + re-exports and reject restoring the old ECStore-owned helper definitions. +- Docs changes: record the object helper contract extraction slice. ## Phase 0 Tasks @@ -536,6 +537,26 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block migration/layer guards, formatting, diff hygiene, Rust risk scan, and required three-expert review passed. +- [x] `API-015` Move object option helper contracts. + - Completed slice: move `CompletePart`, `HTTPPreconditions`, and + `ObjectLockRetentionOptions` from ECStore `store_api` into + `rustfs-storage-api`; keep `ObjectOptions`, object/list DTOs, readers, + filemeta conversions, and storage implementations in ECStore. + - Acceptance: `rustfs-storage-api` exports the moved helper contracts, + in-repo consumers no longer use the old `rustfs_ecstore::store_api` path + for these helpers, and migration guards reject restoring the old ECStore + definitions or public re-exports. + - Must preserve: multipart completion mapping, HTTP precondition semantics, + object-lock retention fields, object lookup/drop-precondition behavior, + storage hot paths, and ECStore-owned implementation-heavy object contracts. + - Risk defense: only pure helper DTOs cross into `rustfs-storage-api`; + ECStore keeps `ObjectOptions`, `ObjectInfo`, list contracts, readers, + lifecycle/replication/rio/filemeta coupling, errors, and implementation + bodies. + - Verification: focused storage-api/ECStore/RustFS/downstream compile checks, + migration/layer guards, formatting, diff hygiene, Rust risk scan, and + required three-expert review passed. + ## Phase 8 Background Controller Tasks - [x] `BGC-001` Inventory background services. @@ -812,16 +833,15 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | `BucketOperations` now lives in the storage contract crate with an associated ECStore error type, while implementation bodies and storage-heavy contracts remain in ECStore. | -| Migration preservation | passed | The slice changes trait ownership and import paths only; bucket operation bodies, metadata migration flow, admin/Swift/storage callers, and storage hot paths are unchanged. | +| Quality/architecture | passed | Pure multipart/object helper contracts now live in `rustfs-storage-api`; implementation-heavy object contracts and ECStore behavior stay in ECStore. | +| Migration preservation | passed | The slice changes helper type ownership and import paths only; multipart completion mapping, HTTP precondition values, object-lock retention fields, and storage hot paths are unchanged. | | Testing/verification | passed | Focused storage-api/ECStore/RustFS/downstream compile checks, migration/layer guards, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` passed. | ## Verification Notes -Passed on `1830911973f882e30a1de18a5ff7d2ab669d67dc`: +Passed on `8d24d9133b31c0ca0de0fcd16ad06cd2fdc6f7ae`: -- `cargo check -p rustfs-storage-api -p rustfs-ecstore`: passed. -- `cargo check -p rustfs -p rustfs-heal -p rustfs-scanner -p rustfs-protocols -p rustfs-obs`: passed. +- `cargo check -p rustfs-storage-api -p rustfs-ecstore -p rustfs-iam -p rustfs-scanner -p rustfs`: passed. - `./scripts/check_architecture_migration_rules.sh`: passed. - `./scripts/check_layer_dependencies.sh`: passed. - `cargo fmt --all --check`: passed. @@ -832,16 +852,17 @@ Passed on `1830911973f882e30a1de18a5ff7d2ab669d67dc`: Notes: -- This slice follows `rustfs/rustfs#3505` and keeps the old aggregate facade, - bucket DTO, multipart DTO, and bucket operation contract guards active. -- The shared `BucketOperations` contract is now owned by - `rustfs-storage-api`; ECStore keeps implementation-heavy object, list, - reader, lock, and storage error contracts. -- The slice does not alter bucket runtime behavior. +- This slice follows `rustfs/rustfs#3507` and keeps the old aggregate facade, + bucket DTO, multipart DTO, bucket operation contract, and object helper + contract guards active. +- The shared object helper contracts are now owned by `rustfs-storage-api`; + ECStore keeps implementation-heavy object, list, reader, lock, lifecycle, + replication, rio, filemeta, and storage error contracts. +- The slice does not alter object, multipart, or bucket runtime behavior. ## Handoff Notes -- Bucket operation contract cleanup is in progress on a branch current with +- Object helper contract cleanup is in progress on a branch current with `origin/main`. -- After this lands, remaining storage work can continue by extracting only DTOs - that have proven low implementation coupling. +- After this lands, remaining storage work can continue by extracting larger + low-coupling DTO slices or by narrowing remaining operation-group consumers. diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 765de09d9..6385738f7 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -53,11 +53,13 @@ use rustfs_ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_r use rustfs_ecstore::rio::{HashReader, WritePlan}; use rustfs_ecstore::set_disk::is_valid_storage_class; use rustfs_ecstore::store::ECStore; -use rustfs_ecstore::store_api::{CompletePart, HTTPRangeSpec, ObjectIO, ObjectOptions, PutObjReader}; +use rustfs_ecstore::store_api::{HTTPRangeSpec, ObjectIO, ObjectOptions, PutObjReader}; use rustfs_ecstore::store_api::{MultipartOperations, ObjectOperations}; use rustfs_filemeta::{ReplicationStatusType, ReplicationType}; use rustfs_s3_ops::S3Operation; -use rustfs_storage_api::MultipartUploadResult; +#[cfg(test)] +use rustfs_storage_api::HTTPPreconditions; +use rustfs_storage_api::{CompletePart, MultipartUploadResult}; use rustfs_targets::EventName; use rustfs_utils::CompressionAlgorithm; use rustfs_utils::http::{ @@ -146,6 +148,21 @@ fn normalize_complete_multipart_parts(parts: Vec) -> S3Result CompletePart { + CompletePart { + part_num: value + .part_number + .and_then(|part_num| usize::try_from(part_num).ok()) + .unwrap_or_default(), + etag: value.e_tag.map(|v| v.value().to_owned()), + checksum_crc32: value.checksum_crc32, + checksum_crc32c: value.checksum_crc32c, + checksum_sha1: value.checksum_sha1, + checksum_sha256: value.checksum_sha256, + checksum_crc64nvme: value.checksum_crc64nvme, + } +} + async fn validate_table_catalog_object_mutation(bucket: &str, key: &str) -> S3Result<()> { table_catalog::validate_bucket_object_mutation(bucket, key) .await @@ -380,7 +397,7 @@ impl DefaultMultipartUsecase { .parts .unwrap_or_default() .into_iter() - .map(CompletePart::from) + .map(complete_part_from_s3) .collect::>(); let uploaded_parts = normalize_complete_multipart_parts(uploaded_parts_vec)?; @@ -1403,7 +1420,7 @@ mod tests { let opts = ObjectOptions { version_id: Some(Uuid::new_v4().to_string()), no_lock: true, - http_preconditions: Some(rustfs_ecstore::store_api::HTTPPreconditions { + http_preconditions: Some(HTTPPreconditions { if_none_match: Some("*".to_string()), if_match: Some("\"etag\"".to_string()), ..Default::default() diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 7b3b6d482..7520ed325 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -93,6 +93,8 @@ use rustfs_notify::EventArgsBuilder; use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_for_post_object}; use rustfs_s3select_api::object_store::bytes_stream; +#[cfg(test)] +use rustfs_storage_api::HTTPPreconditions; use rustfs_targets::{ EventName, extract_params_header, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent, }; @@ -4897,7 +4899,7 @@ mod tests { let opts = ObjectOptions { version_id: Some(version_id.clone()), no_lock: true, - http_preconditions: Some(rustfs_ecstore::store_api::HTTPPreconditions { + http_preconditions: Some(HTTPPreconditions { if_none_match: Some("\"etag\"".to_string()), if_match: Some("\"other\"".to_string()), ..Default::default() diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index d6e71732a..8af171766 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -40,11 +40,11 @@ use rustfs_ecstore::{ versioning_sys::BucketVersioningSys, }, error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found}, - store_api::{ObjectLockRetentionOptions, ObjectOperations, ObjectOptions}, + store_api::{ObjectOperations, ObjectOptions}, }; use rustfs_io_metrics::record_s3_op; use rustfs_s3_ops::S3Operation; -use rustfs_storage_api::{BucketOperations, BucketOptions}; +use rustfs_storage_api::{BucketOperations, BucketOptions, ObjectLockRetentionOptions}; use rustfs_targets::EventName; use rustfs_utils::http::headers::{ AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index db9d519c2..86401771d 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -31,8 +31,9 @@ use s3s::header::X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE; use crate::auth::UNSIGNED_PAYLOAD; use crate::auth::UNSIGNED_PAYLOAD_TRAILER; -use rustfs_ecstore::store_api::{HTTPPreconditions, HTTPRangeSpec, ObjectOptions}; +use rustfs_ecstore::store_api::{HTTPRangeSpec, ObjectOptions}; use rustfs_policy::service_type::ServiceType; +use rustfs_storage_api::HTTPPreconditions; use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH; use rustfs_utils::http::AMZ_CONTENT_SHA256; use rustfs_utils::path::is_dir_object; diff --git a/rustfs/src/table_catalog.rs b/rustfs/src/table_catalog.rs index 580a8d465..9f0c83275 100644 --- a/rustfs/src/table_catalog.rs +++ b/rustfs/src/table_catalog.rs @@ -45,8 +45,9 @@ use rustfs_ecstore::disk::RUSTFS_META_BUCKET; use rustfs_ecstore::error::StorageError; use rustfs_ecstore::{ set_disk::get_lock_acquire_timeout, - store_api::{HTTPPreconditions, ListOperations, NamespaceLocking, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}, + store_api::{ListOperations, NamespaceLocking, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}, }; +use rustfs_storage_api::HTTPPreconditions; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use time::{Duration, OffsetDateTime}; use tokio::io::AsyncReadExt; diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 4aff14f46..6e51a3d8f 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -53,6 +53,7 @@ LEGACY_STORAGE_API_HITS_FILE="${TMP_DIR}/legacy_storage_api_hits.txt" STORE_API_BUCKET_DTO_REEXPORTS_FILE="${TMP_DIR}/store_api_bucket_dto_reexports.txt" STORE_API_BUCKET_OPERATION_HITS_FILE="${TMP_DIR}/store_api_bucket_operation_hits.txt" STORE_API_MULTIPART_DTO_REEXPORTS_FILE="${TMP_DIR}/store_api_multipart_dto_reexports.txt" +STORE_API_OBJECT_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_object_helper_reexports.txt" awk ' /^## PR Types$/ { @@ -187,8 +188,12 @@ require_source_line \ "storage-api public bucket contract re-export" require_source_line \ "crates/storage-api/src/lib.rs" \ - "pub use multipart::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};" \ + "pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};" \ "storage-api public multipart DTO re-export" +require_source_line \ + "crates/storage-api/src/lib.rs" \ + "pub use object::{HTTPPreconditions, ObjectLockRetentionOptions};" \ + "storage-api public object helper contract re-export" require_source_line \ "crates/storage-api/src/lib.rs" \ "pub use error::{StorageErrorCode, StorageResult};" \ @@ -225,7 +230,7 @@ fi ( cd "$ROOT_DIR" - rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api::\{[^}]*\b(?:ListMultipartsInfo|ListPartsInfo|MultipartInfo|MultipartUploadResult|PartInfo)\b|pub struct (?:ListMultipartsInfo|ListPartsInfo|MultipartInfo|MultipartUploadResult|PartInfo)\b' \ + rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api::\{[^}]*\b(?:CompletePart|ListMultipartsInfo|ListPartsInfo|MultipartInfo|MultipartUploadResult|PartInfo)\b|pub struct (?:CompletePart|ListMultipartsInfo|ListPartsInfo|MultipartInfo|MultipartUploadResult|PartInfo)\b' \ crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true ) >"$STORE_API_MULTIPART_DTO_REEXPORTS_FILE" @@ -233,6 +238,16 @@ if [[ -s "$STORE_API_MULTIPART_DTO_REEXPORTS_FILE" ]]; then report_failure "old ecstore store_api multipart DTO path reintroduced: $(paste -sd '; ' "$STORE_API_MULTIPART_DTO_REEXPORTS_FILE")" fi +( + cd "$ROOT_DIR" + rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api::\{[^}]*\b(?:HTTPPreconditions|ObjectLockRetentionOptions)\b|pub struct (?:HTTPPreconditions|ObjectLockRetentionOptions)\b' \ + crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true +) >"$STORE_API_OBJECT_HELPER_REEXPORTS_FILE" + +if [[ -s "$STORE_API_OBJECT_HELPER_REEXPORTS_FILE" ]]; then + report_failure "old ecstore store_api object helper path reintroduced: $(paste -sd '; ' "$STORE_API_OBJECT_HELPER_REEXPORTS_FILE")" +fi + require_source_contains \ "crates/ecstore/src/store_api/traits.rs" \ "pub trait NamespaceLocking: Send + Sync + Debug + 'static" \