From af6c32efac3d0860d7240149c8db8798b39632fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Sat, 28 Feb 2026 01:19:47 +0800 Subject: [PATCH] refactor: improve code quality with safer error handling, trait decomposition, and dead code cleanup (#1997) --- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 2 +- crates/ecstore/src/bucket/metadata_sys.rs | 2 +- .../src/client/object_handlers_common.rs | 3 +- crates/ecstore/src/data_usage.rs | 8 +- crates/ecstore/src/error.rs | 348 ++++++--------- crates/ecstore/src/pools.rs | 3 +- crates/ecstore/src/rebalance.rs | 4 +- crates/ecstore/src/set_disk.rs | 151 ++++--- crates/ecstore/src/sets.rs | 333 +++++++------- crates/ecstore/src/store.rs | 256 ++++++----- crates/ecstore/src/store_api/traits.rs | 135 +++--- crates/ecstore/src/store_list_objects.rs | 4 +- crates/heal/src/heal/storage.rs | 2 +- crates/heal/tests/heal_integration_test.rs | 2 +- crates/iam/src/store/object.rs | 3 +- crates/metrics/src/collectors/global.rs | 2 +- crates/policy/src/policy/variables.rs | 2 +- crates/s3select-api/src/object_store.rs | 2 +- crates/scanner/src/scanner_io.rs | 2 +- .../tests/lifecycle_integration_test.rs | 2 +- rustfs/src/admin/handlers/account_info.rs | 2 +- rustfs/src/admin/handlers/bucket_meta.rs | 11 +- rustfs/src/admin/handlers/rebalance.rs | 1 + rustfs/src/admin/handlers/replication.rs | 2 +- rustfs/src/app/admin_usecase.rs | 52 +-- rustfs/src/app/bucket_usecase.rs | 121 +----- rustfs/src/app/context.rs | 7 +- rustfs/src/app/multipart_usecase.rs | 136 +----- rustfs/src/app/object_usecase.rs | 407 ++++++------------ rustfs/src/config/workload_profiles.rs | 5 +- rustfs/src/license.rs | 15 - rustfs/src/main.rs | 19 +- rustfs/src/server/http.rs | 9 +- rustfs/src/server/runtime.rs | 30 +- rustfs/src/storage/access.rs | 139 +++--- rustfs/src/storage/concurrency.rs | 17 +- .../src/storage/concurrent_get_object_test.rs | 2 +- rustfs/src/storage/ecfs.rs | 3 +- rustfs/src/storage/ecfs_extend.rs | 4 +- rustfs/src/storage/entity.rs | 3 +- rustfs/src/storage/head_prefix.rs | 2 +- rustfs/src/storage/rpc/disk.rs | 33 -- rustfs/src/storage/rpc/node_service.rs | 208 --------- rustfs/src/storage/s3_api/mod.rs | 3 +- rustfs/src/storage/sse.rs | 27 +- scripts/layer-dependency-baseline.txt | 27 +- 46 files changed, 939 insertions(+), 1612 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index af0ad391d..0e370ec69 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -34,7 +34,7 @@ use crate::global::GLOBAL_LocalNodeName; use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id}; use crate::store::ECStore; use crate::store_api::StorageAPI; -use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete}; +use crate::store_api::{GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete}; use crate::tier::warm_backend::WarmBackendGetOpts; use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded}; use bytes::BytesMut; diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 95c19d024..531cefcea 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -15,13 +15,13 @@ use super::metadata::{BucketMetadata, load_bucket_metadata}; use super::quota::BucketQuota; use super::target::BucketTargets; -use crate::StorageAPI as _; use crate::bucket::bucket_target_sys::BucketTargetSys; use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, load_bucket_metadata_parse}; use crate::bucket::utils::{deserialize, is_meta_bucketname}; use crate::error::{Error, Result, is_err_bucket_not_found}; use crate::global::{GLOBAL_Endpoints, is_dist_erasure, is_erasure, new_object_layer_fn}; use crate::store::ECStore; +use crate::store_api::HealOperations as _; use futures::future::join_all; use lazy_static::lazy_static; use rustfs_common::heal_channel::HealOpts; diff --git a/crates/ecstore/src/client/object_handlers_common.rs b/crates/ecstore/src/client/object_handlers_common.rs index 5c8a110cb..41e68134c 100644 --- a/crates/ecstore/src/client/object_handlers_common.rs +++ b/crates/ecstore/src/client/object_handlers_common.rs @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::StorageAPI; use crate::bucket::lifecycle::lifecycle; use crate::bucket::versioning::VersioningApi; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::store::ECStore; -use crate::store_api::{ObjectOptions, ObjectToDelete}; +use crate::store_api::{ObjectOperations, ObjectOptions, ObjectToDelete}; use rustfs_lock::MAX_DELETE_LIST; pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) { diff --git a/crates/ecstore/src/data_usage.rs b/crates/ecstore/src/data_usage.rs index f23c2b85f..cfcad4900 100644 --- a/crates/ecstore/src/data_usage.rs +++ b/crates/ecstore/src/data_usage.rs @@ -15,8 +15,12 @@ pub mod local_snapshot; use crate::{ - bucket::metadata_sys::get_replication_config, config::com::read_config, disk::DiskAPI, error::Error, store::ECStore, - store_api::StorageAPI, + bucket::metadata_sys::get_replication_config, + config::com::read_config, + disk::DiskAPI, + error::Error, + store::ECStore, + store_api::{BucketOperations, ListOperations}, }; pub use local_snapshot::{ DATA_USAGE_DIR, DATA_USAGE_STATE_DIR, LOCAL_USAGE_SNAPSHOT_VERSION, LocalUsageSnapshot, LocalUsageSnapshotMeta, diff --git a/crates/ecstore/src/error.rs b/crates/ecstore/src/error.rs index dc747c36c..7c7ed1f90 100644 --- a/crates/ecstore/src/error.rs +++ b/crates/ecstore/src/error.rs @@ -20,186 +20,156 @@ use s3s::{S3Error, S3ErrorCode}; pub type Error = StorageError; pub type Result = core::result::Result; +/// Storage layer error type covering disk, volume, bucket, object, multipart, +/// erasure-coding, and operational error conditions. +/// +/// Variants are organized by domain: +/// - **Disk / Volume** – low-level storage I/O +/// - **Bucket** – bucket-level operations +/// - **Object** – object-level operations (including versioning) +/// - **Multipart** – multipart upload lifecycle +/// - **Erasure / Quorum** – erasure coding and quorum failures +/// - **Operational** – decommission, healing, rate-limiting, etc. +/// - **Generic** – I/O, locks, and catch-all errors #[derive(Debug, thiserror::Error)] pub enum StorageError { + // ── Disk / Volume ──────────────────────────────────────────────── #[error("Faulty disk")] FaultyDisk, - - #[error("Disk full")] - DiskFull, - - #[error("Volume not found")] - VolumeNotFound, - - #[error("Volume exists")] - VolumeExists, - - #[error("File not found")] - FileNotFound, - - #[error("File version not found")] - FileVersionNotFound, - - #[error("File name too long")] - FileNameTooLong, - - #[error("File access denied")] - FileAccessDenied, - - #[error("File is corrupted")] - FileCorrupt, - - #[error("Not a regular file")] - IsNotRegular, - - #[error("Volume not empty")] - VolumeNotEmpty, - - #[error("Volume access denied")] - VolumeAccessDenied, - - #[error("Corrupted format")] - CorruptedFormat, - - #[error("Corrupted backend")] - CorruptedBackend, - - #[error("Unformatted disk")] - UnformattedDisk, - - #[error("Disk not found")] - DiskNotFound, - - #[error("Drive is root")] - DriveIsRoot, - #[error("Faulty remote disk")] FaultyRemoteDisk, - + #[error("Disk full")] + DiskFull, + #[error("Disk not found")] + DiskNotFound, #[error("Disk access denied")] DiskAccessDenied, - - #[error("Unexpected error")] - Unexpected, - + #[error("Drive is root")] + DriveIsRoot, + #[error("Unformatted disk")] + UnformattedDisk, + #[error("Corrupted format")] + CorruptedFormat, + #[error("Corrupted backend")] + CorruptedBackend, #[error("Too many open files")] TooManyOpenFiles, - #[error("No heal required")] - NoHealRequired, + #[error("Volume not found")] + VolumeNotFound, + #[error("Volume exists")] + VolumeExists, + #[error("Volume not empty")] + VolumeNotEmpty, + #[error("Volume access denied")] + VolumeAccessDenied, - #[error("Config not found")] - ConfigNotFound, - - #[error("not implemented")] - NotImplemented, - - #[error("Invalid arguments provided for {0}/{1}-{2}")] - InvalidArgument(String, String, String), - - #[error("method not allowed")] - MethodNotAllowed, + #[error("File not found")] + FileNotFound, + #[error("File version not found")] + FileVersionNotFound, + #[error("File name too long")] + FileNameTooLong, + #[error("File access denied")] + FileAccessDenied, + #[error("File is corrupted")] + FileCorrupt, + #[error("Not a regular file")] + IsNotRegular, + // ── Bucket ─────────────────────────────────────────────────────── #[error("Bucket not found: {0}")] BucketNotFound(String), - + #[error("Bucket exists: {0}")] + BucketExists(String), #[error("Bucket not empty: {0}")] BucketNotEmpty(String), - #[error("Bucket name invalid: {0}")] BucketNameInvalid(String), - #[error("Object name invalid: {0}/{1}")] - ObjectNameInvalid(String, String), - - #[error("Bucket exists: {0}")] - BucketExists(String), - #[error("Storage reached its minimum free drive threshold.")] - StorageFull, - #[error("Please reduce your request rate")] - SlowDown, - - #[error("Prefix access is denied:{0}/{1}")] - PrefixAccessDenied(String, String), - - #[error("Invalid UploadID KeyCombination: {0}/{1}")] - InvalidUploadIDKeyCombination(String, String), - - #[error("Malformed UploadID: {0}")] - MalformedUploadID(String), - - #[error("Object name too long: {0}/{1}")] - ObjectNameTooLong(String, String), - - #[error("Object name contains forward slash as prefix: {0}/{1}")] - ObjectNamePrefixAsSlash(String, String), - + // ── Object ─────────────────────────────────────────────────────── #[error("Object not found: {0}/{1}")] ObjectNotFound(String, String), - + #[error("Object name invalid: {0}/{1}")] + ObjectNameInvalid(String, String), + #[error("Object name too long: {0}/{1}")] + ObjectNameTooLong(String, String), + #[error("Object name contains forward slash as prefix: {0}/{1}")] + ObjectNamePrefixAsSlash(String, String), + #[error("Object exists on :{0} as directory {1}")] + ObjectExistsAsDirectory(String, String), #[error("Version not found: {0}/{1}-{2}")] VersionNotFound(String, String, String), - - #[error("Invalid upload id: {0}/{1}-{2}")] - InvalidUploadID(String, String, String), - - #[error("Specified part could not be found. PartNumber {0}, Expected {1}, got {2}")] - InvalidPart(usize, String, String), - - #[error("Your proposed upload is smaller than the minimum allowed size. Part {0} size {1} is less than minimum {2}")] - EntityTooSmall(usize, i64, i64), - #[error("Invalid version id: {0}/{1}-{2}")] InvalidVersionID(String, String, String), #[error("invalid data movement operation, source and destination pool are the same for : {0}/{1}-{2}")] DataMovementOverwriteErr(String, String, String), + #[error("Prefix access is denied:{0}/{1}")] + PrefixAccessDenied(String, String), - #[error("Object exists on :{0} as directory {1}")] - ObjectExistsAsDirectory(String, String), + // ── Multipart ──────────────────────────────────────────────────── + #[error("Invalid upload id: {0}/{1}-{2}")] + InvalidUploadID(String, String, String), + #[error("Invalid UploadID KeyCombination: {0}/{1}")] + InvalidUploadIDKeyCombination(String, String), + #[error("Malformed UploadID: {0}")] + MalformedUploadID(String), + #[error("Specified part could not be found. PartNumber {0}, Expected {1}, got {2}")] + InvalidPart(usize, String, String), + #[error("Invalid part number: {0}")] + InvalidPartNumber(usize), + #[error("Your proposed upload is smaller than the minimum allowed size. Part {0} size {1} is less than minimum {2}")] + EntityTooSmall(usize, i64, i64), + // ── Erasure / Quorum ───────────────────────────────────────────── + #[error("erasure read quorum")] + ErasureReadQuorum, + #[error("erasure write quorum")] + ErasureWriteQuorum, #[error("Storage resources are insufficient for the read operation: {0}/{1}")] InsufficientReadQuorum(String, String), - #[error("Storage resources are insufficient for the write operation: {0}/{1}")] InsufficientWriteQuorum(String, String), + #[error("not first disk")] + NotFirstDisk, + #[error("first disk wait")] + FirstDiskWait, + // ── Operational ────────────────────────────────────────────────── + #[error("Storage reached its minimum free drive threshold.")] + StorageFull, + #[error("Please reduce your request rate")] + SlowDown, #[error("Decommission not started")] DecommissionNotStarted, #[error("Decommission already running")] DecommissionAlreadyRunning, - + #[error("No heal required")] + NoHealRequired, #[error("DoneForNow")] DoneForNow, - - #[error("erasure read quorum")] - ErasureReadQuorum, - - #[error("erasure write quorum")] - ErasureWriteQuorum, - - #[error("not first disk")] - NotFirstDisk, - - #[error("first disk wait")] - FirstDiskWait, - - #[error("Io error: {0}")] - Io(std::io::Error), - - #[error("Lock error: {0}")] - Lock(#[from] rustfs_lock::LockError), - + #[error("Config not found")] + ConfigNotFound, #[error("Precondition failed")] PreconditionFailed, - #[error("Not modified")] NotModified, - - #[error("Invalid part number: {0}")] - InvalidPartNumber(usize), - #[error("Invalid range specified: {0}")] InvalidRangeSpec(String), + + // ── Generic ────────────────────────────────────────────────────── + #[error("Unexpected error")] + Unexpected, + #[error("not implemented")] + NotImplemented, + #[error("Invalid arguments provided for {0}/{1}-{2}")] + InvalidArgument(String, String, String), + #[error("method not allowed")] + MethodNotAllowed, + #[error("Io error: {0}")] + Io(std::io::Error), + #[error("Lock error: {0}")] + Lock(#[from] rustfs_lock::LockError), } impl StorageError { @@ -209,6 +179,29 @@ impl StorageError { { StorageError::Io(std::io::Error::other(error)) } + + pub fn is_not_found(&self) -> bool { + matches!( + self, + StorageError::FileNotFound + | StorageError::ObjectNotFound(_, _) + | StorageError::FileVersionNotFound + | StorageError::VersionNotFound(_, _, _) + | StorageError::VolumeNotFound + | StorageError::DiskNotFound + | StorageError::BucketNotFound(_) + ) + } + + pub fn is_quorum_error(&self) -> bool { + matches!( + self, + StorageError::ErasureReadQuorum + | StorageError::ErasureWriteQuorum + | StorageError::InsufficientReadQuorum(_, _) + | StorageError::InsufficientWriteQuorum(_, _) + ) + } } impl From for StorageError { @@ -724,24 +717,6 @@ pub fn is_all_volume_not_found(errs: &[Option]) -> bool { !errs.is_empty() } -// pub fn is_all_not_found(errs: &[Option]) -> bool { -// for err in errs.iter() { -// if let Some(err) = err { -// if let Some(err) = err.downcast_ref::() { -// match err { -// DiskError::FileNotFound | DiskError::VolumeNotFound | &DiskError::FileVersionNotFound => { -// continue; -// } -// _ => return false, -// } -// } -// } -// return false; -// } - -// !errs.is_empty() -// } - pub fn to_object_err(err: Error, params: Vec<&str>) -> Error { match err { StorageError::DiskFull => StorageError::StorageFull, @@ -938,10 +913,6 @@ pub fn storage_to_object_err(err: Error, params: Vec<&str>) -> S3Error { object = decode_dir_object(params[1]); } match storage_err { - /*StorageError::NotImplemented => s3_error!(NotImplemented), - StorageError::InvalidArgument(bucket, object, version_id) => { - s3_error!(InvalidArgument, "Invalid arguments provided for {}/{}-{}", bucket, object, version_id) - }*/ StorageError::MethodNotAllowed => S3Error::with_message( S3ErrorCode::MethodNotAllowed, ObjectApiError::MethodNotAllowed(GenericError { @@ -951,73 +922,6 @@ pub fn storage_to_object_err(err: Error, params: Vec<&str>) -> S3Error { }) .to_string(), ), - /*StorageError::BucketNotFound(bucket) => { - s3_error!(NoSuchBucket, "bucket not found {}", bucket) - } - StorageError::BucketNotEmpty(bucket) => s3_error!(BucketNotEmpty, "bucket not empty {}", bucket), - StorageError::BucketNameInvalid(bucket) => s3_error!(InvalidBucketName, "invalid bucket name {}", bucket), - StorageError::ObjectNameInvalid(bucket, object) => { - s3_error!(InvalidArgument, "invalid object name {}/{}", bucket, object) - } - StorageError::BucketExists(bucket) => s3_error!(BucketAlreadyExists, "{}", bucket), - StorageError::StorageFull => s3_error!(ServiceUnavailable, "Storage reached its minimum free drive threshold."), - StorageError::SlowDown => s3_error!(SlowDown, "Please reduce your request rate"), - StorageError::PrefixAccessDenied(bucket, object) => { - s3_error!(AccessDenied, "PrefixAccessDenied {}/{}", bucket, object) - } - StorageError::InvalidUploadIDKeyCombination(bucket, object) => { - s3_error!(InvalidArgument, "Invalid UploadID KeyCombination: {}/{}", bucket, object) - } - StorageError::MalformedUploadID(bucket) => s3_error!(InvalidArgument, "Malformed UploadID: {}", bucket), - StorageError::ObjectNameTooLong(bucket, object) => { - s3_error!(InvalidArgument, "Object name too long: {}/{}", bucket, object) - } - StorageError::ObjectNamePrefixAsSlash(bucket, object) => { - s3_error!(InvalidArgument, "Object name contains forward slash as prefix: {}/{}", bucket, object) - } - StorageError::ObjectNotFound(bucket, object) => s3_error!(NoSuchKey, "{}/{}", bucket, object), - StorageError::VersionNotFound(bucket, object, version_id) => { - s3_error!(NoSuchVersion, "{}/{}/{}", bucket, object, version_id) - } - StorageError::InvalidUploadID(bucket, object, version_id) => { - s3_error!(InvalidPart, "Invalid upload id: {}/{}-{}", bucket, object, version_id) - } - StorageError::InvalidVersionID(bucket, object, version_id) => { - s3_error!(InvalidArgument, "Invalid version id: {}/{}-{}", bucket, object, version_id) - } - // extended - StorageError::DataMovementOverwriteErr(bucket, object, version_id) => s3_error!( - InvalidArgument, - "invalid data movement operation, source and destination pool are the same for : {}/{}-{}", - bucket, - object, - version_id - ), - // extended - StorageError::ObjectExistsAsDirectory(bucket, object) => { - s3_error!(InvalidArgument, "Object exists on :{} as directory {}", bucket, object) - } - StorageError::InsufficientReadQuorum => { - s3_error!(SlowDown, "Storage resources are insufficient for the read operation") - } - StorageError::InsufficientWriteQuorum => { - s3_error!(SlowDown, "Storage resources are insufficient for the write operation") - } - StorageError::DecommissionNotStarted => s3_error!(InvalidArgument, "Decommission Not Started"), - - StorageError::VolumeNotFound(bucket) => { - s3_error!(NoSuchBucket, "bucket not found {}", bucket) - } - StorageError::InvalidPart(bucket, object, version_id) => { - s3_error!( - InvalidPart, - "Specified part could not be found. PartNumber {}, Expected {}, got {}", - bucket, - object, - version_id - ) - } - StorageError::DoneForNow => s3_error!(InternalError, "DoneForNow"),*/ _ => s3s::S3Error::with_message(S3ErrorCode::Custom("err".into()), err.to_string()), } } diff --git a/crates/ecstore/src/pools.rs b/crates/ecstore/src/pools.rs index 65950743a..2d9222cae 100644 --- a/crates/ecstore/src/pools.rs +++ b/crates/ecstore/src/pools.rs @@ -27,7 +27,8 @@ use crate::new_object_layer_fn; use crate::notification_sys::get_global_notification_sys; use crate::set_disk::SetDisks; use crate::store_api::{ - BucketOptions, CompletePart, GetObjectReader, MakeBucketOptions, ObjectIO, ObjectOptions, PutObjReader, StorageAPI, + BucketOperations, BucketOptions, CompletePart, GetObjectReader, HealOperations, MakeBucketOptions, MultipartOperations, + ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, StorageAPI, }; use crate::{sets::Sets, store::ECStore}; use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; diff --git a/crates/ecstore/src/rebalance.rs b/crates/ecstore/src/rebalance.rs index 5fe758257..79621de0b 100644 --- a/crates/ecstore/src/rebalance.rs +++ b/crates/ecstore/src/rebalance.rs @@ -22,7 +22,9 @@ use crate::global::get_global_endpoints; use crate::pools::ListCallback; use crate::set_disk::SetDisks; use crate::store::ECStore; -use crate::store_api::{CompletePart, GetObjectReader, ObjectIO, ObjectOptions, PutObjReader}; +use crate::store_api::{ + CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader, +}; use http::HeaderMap; use rustfs_common::defer; use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 571fcdb71..87df89ed2 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -52,9 +52,9 @@ use crate::{ event_notification::{EventArgs, send_event}, global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure}, store_api::{ - BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec, - ListMultipartsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectIO, ObjectInfo, - PartInfo, PutObjReader, StorageAPI, + BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, + HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectsV2Info, ListOperations, MakeBucketOptions, MultipartInfo, + MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOperations, PartInfo, PutObjReader, StorageAPI, }, store_init::load_format_erasure, }; @@ -842,10 +842,20 @@ impl StorageAPI for SetDisks { get_storage_info(&local_disks, &local_endpoints).await } + #[tracing::instrument(skip(self))] - async fn list_bucket(&self, _opts: &BucketOptions) -> Result> { + async fn get_disks(&self, _pool_idx: usize, _set_idx: usize) -> Result>> { + Ok(self.get_disks_internal().await) + } + + #[tracing::instrument(skip(self))] + fn set_drive_counts(&self) -> Vec { unimplemented!() } +} + +#[async_trait::async_trait] +impl BucketOperations for SetDisks { #[tracing::instrument(skip(self))] async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> Result<()> { unimplemented!() @@ -856,6 +866,19 @@ impl StorageAPI for SetDisks { unimplemented!() } + #[tracing::instrument(skip(self))] + async fn list_bucket(&self, _opts: &BucketOptions) -> Result> { + unimplemented!() + } + + #[tracing::instrument(skip(self))] + async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> { + unimplemented!() + } +} + +#[async_trait::async_trait] +impl ObjectOperations for SetDisks { #[tracing::instrument(skip(self))] async fn copy_object( &self, @@ -1400,45 +1423,6 @@ impl StorageAPI for SetDisks { Ok(obj_info) } - #[tracing::instrument(skip(self))] - async fn list_objects_v2( - self: Arc, - _bucket: &str, - _prefix: &str, - _continuation_token: Option, - _delimiter: Option, - _max_keys: i32, - _fetch_owner: bool, - _start_after: Option, - _incl_deleted: bool, - ) -> Result { - unimplemented!() - } - - #[tracing::instrument(skip(self))] - async fn list_object_versions( - self: Arc, - _bucket: &str, - _prefix: &str, - _marker: Option, - _version_marker: Option, - _delimiter: Option, - _max_keys: i32, - ) -> Result { - unimplemented!() - } - - async fn walk( - self: Arc, - _rx: CancellationToken, - _bucket: &str, - _prefix: &str, - _result: Sender, - _opts: WalkOptions, - ) -> Result<()> { - unimplemented!() - } - #[tracing::instrument(skip(self))] async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { // Acquire a shared read-lock to protect consistency during info fetch @@ -1890,6 +1874,60 @@ impl StorageAPI for SetDisks { self.put_object_tags(bucket, object, "", opts).await } + #[tracing::instrument(skip(self))] + async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let get_object_reader = ::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?; + // Stream to sink to avoid loading entire object into memory during verification + let mut reader = get_object_reader.stream; + tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?; + Ok(()) + } +} + +#[async_trait::async_trait] +impl ListOperations for SetDisks { + #[tracing::instrument(skip(self))] + async fn list_objects_v2( + self: Arc, + _bucket: &str, + _prefix: &str, + _continuation_token: Option, + _delimiter: Option, + _max_keys: i32, + _fetch_owner: bool, + _start_after: Option, + _incl_deleted: bool, + ) -> Result { + unimplemented!() + } + + #[tracing::instrument(skip(self))] + async fn list_object_versions( + self: Arc, + _bucket: &str, + _prefix: &str, + _marker: Option, + _version_marker: Option, + _delimiter: Option, + _max_keys: i32, + ) -> Result { + unimplemented!() + } + + async fn walk( + self: Arc, + _rx: CancellationToken, + _bucket: &str, + _prefix: &str, + _result: Sender, + _opts: WalkOptions, + ) -> Result<()> { + unimplemented!() + } +} + +#[async_trait::async_trait] +impl MultipartOperations for SetDisks { #[tracing::instrument(skip(self))] async fn copy_object_part( &self, @@ -2922,22 +2960,10 @@ impl StorageAPI for SetDisks { Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended)) } +} - #[tracing::instrument(skip(self))] - async fn get_disks(&self, _pool_idx: usize, _set_idx: usize) -> Result>> { - Ok(self.get_disks_internal().await) - } - - #[tracing::instrument(skip(self))] - fn set_drive_counts(&self) -> Vec { - unimplemented!() - } - - #[tracing::instrument(skip(self))] - async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> { - unimplemented!() - } - +#[async_trait::async_trait] +impl HealOperations for SetDisks { #[tracing::instrument(skip(self))] async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option)> { unimplemented!() @@ -3023,15 +3049,6 @@ impl StorageAPI for SetDisks { async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> { unimplemented!() } - - #[tracing::instrument(skip(self))] - async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { - let get_object_reader = ::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?; - // Stream to sink to avoid loading entire object into memory during verification - let mut reader = get_object_reader.stream; - tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?; - Ok(()) - } } #[derive(Debug, PartialEq, Eq)] diff --git a/crates/ecstore/src/sets.rs b/crates/ecstore/src/sets.rs index 895d26149..b984ad721 100644 --- a/crates/ecstore/src/sets.rs +++ b/crates/ecstore/src/sets.rs @@ -28,9 +28,10 @@ use crate::{ global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure}, set_disk::SetDisks, store_api::{ - BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec, - ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartInfo, MultipartUploadResult, - ObjectIO, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, + BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, + HTTPRangeSpec, HealOperations, ListMultipartsInfo, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, + MakeBucketOptions, MultipartInfo, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectInfo, ObjectOperations, + ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, }, store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file}, }; @@ -357,56 +358,7 @@ impl ObjectIO for Sets { } #[async_trait::async_trait] -impl StorageAPI for Sets { - #[tracing::instrument(skip(self))] - async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result { - self.disk_set[0].new_ns_lock(bucket, object).await - } - #[tracing::instrument(skip(self))] - async fn backend_info(&self) -> rustfs_madmin::BackendInfo { - unimplemented!() - } - #[tracing::instrument(skip(self))] - async fn storage_info(&self) -> rustfs_madmin::StorageInfo { - let mut futures = Vec::with_capacity(self.disk_set.len()); - - for set in self.disk_set.iter() { - futures.push(set.storage_info()) - } - - let results = join_all(futures).await; - - let mut disks = Vec::new(); - - for res in results.into_iter() { - disks.extend_from_slice(&res.disks); - } - - rustfs_madmin::StorageInfo { - disks, - ..Default::default() - } - } - #[tracing::instrument(skip(self))] - async fn local_storage_info(&self) -> rustfs_madmin::StorageInfo { - let mut futures = Vec::with_capacity(self.disk_set.len()); - - for set in self.disk_set.iter() { - futures.push(set.local_storage_info()) - } - - let results = join_all(futures).await; - - let mut disks = Vec::new(); - - for res in results.into_iter() { - disks.extend_from_slice(&res.disks); - } - rustfs_madmin::StorageInfo { - disks, - ..Default::default() - } - } +impl BucketOperations for Sets { #[tracing::instrument(skip(self))] async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> Result<()> { unimplemented!() @@ -425,51 +377,26 @@ impl StorageAPI for Sets { async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> { unimplemented!() } +} - #[tracing::instrument(skip(self))] - async fn list_objects_v2( - self: Arc, - _bucket: &str, - _prefix: &str, - _continuation_token: Option, - _delimiter: Option, - _max_keys: i32, - _fetch_owner: bool, - _start_after: Option, - _incl_deleted: bool, - ) -> Result { - unimplemented!() - } - - #[tracing::instrument(skip(self))] - async fn list_object_versions( - self: Arc, - _bucket: &str, - _prefix: &str, - _marker: Option, - _version_marker: Option, - _delimiter: Option, - _max_keys: i32, - ) -> Result { - unimplemented!() - } - - async fn walk( - self: Arc, - _rx: CancellationToken, - _bucket: &str, - _prefix: &str, - _result: tokio::sync::mpsc::Sender, - _opts: WalkOptions, - ) -> Result<()> { - unimplemented!() - } - +#[async_trait::async_trait] +impl ObjectOperations for Sets { #[tracing::instrument(skip(self))] async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { self.get_disks_by_key(object).get_object_info(bucket, object, opts).await } + #[tracing::instrument(level = "debug", skip(self))] + async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + let gor = self.get_object_reader(bucket, object, None, HeaderMap::new(), opts).await?; + let mut reader = gor.stream; + + // Stream data to sink instead of reading all into memory to prevent OOM + tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?; + + Ok(()) + } + #[tracing::instrument(skip(self))] async fn copy_object( &self, @@ -600,20 +527,90 @@ impl StorageAPI for Sets { (del_objects, del_errs) } - async fn list_object_parts( - &self, - bucket: &str, - object: &str, - upload_id: &str, - part_number_marker: Option, - max_parts: usize, - opts: &ObjectOptions, - ) -> Result { + #[tracing::instrument(skip(self))] + async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { + self.get_disks_by_key(object).put_object_metadata(bucket, object, opts).await + } + + #[tracing::instrument(skip(self))] + async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { + self.get_disks_by_key(object).get_object_tags(bucket, object, opts).await + } + + #[tracing::instrument(level = "debug", skip(self))] + async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result { self.get_disks_by_key(object) - .list_object_parts(bucket, object, upload_id, part_number_marker, max_parts, opts) + .put_object_tags(bucket, object, tags, opts) .await } + #[tracing::instrument(skip(self))] + async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { + self.get_disks_by_key(object).delete_object_tags(bucket, object, opts).await + } + + #[tracing::instrument(skip(self))] + async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { + self.get_disks_by_key(object).add_partial(bucket, object, version_id).await + } + + #[tracing::instrument(skip(self))] + async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + self.get_disks_by_key(object).transition_object(bucket, object, opts).await + } + + #[tracing::instrument(skip(self))] + async fn restore_transitioned_object(self: Arc, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + self.get_disks_by_key(object) + .restore_transitioned_object(bucket, object, opts) + .await + } +} + +#[async_trait::async_trait] +impl ListOperations for Sets { + #[tracing::instrument(skip(self))] + async fn list_objects_v2( + self: Arc, + _bucket: &str, + _prefix: &str, + _continuation_token: Option, + _delimiter: Option, + _max_keys: i32, + _fetch_owner: bool, + _start_after: Option, + _incl_deleted: bool, + ) -> Result { + unimplemented!() + } + + #[tracing::instrument(skip(self))] + async fn list_object_versions( + self: Arc, + _bucket: &str, + _prefix: &str, + _marker: Option, + _version_marker: Option, + _delimiter: Option, + _max_keys: i32, + ) -> Result { + unimplemented!() + } + + async fn walk( + self: Arc, + _rx: CancellationToken, + _bucket: &str, + _prefix: &str, + _result: tokio::sync::mpsc::Sender, + _opts: WalkOptions, + ) -> Result<()> { + unimplemented!() + } +} + +#[async_trait::async_trait] +impl MultipartOperations for Sets { #[tracing::instrument(skip(self))] async fn list_multipart_uploads( &self, @@ -633,23 +630,6 @@ impl StorageAPI for Sets { self.get_disks_by_key(object).new_multipart_upload(bucket, object, opts).await } - #[tracing::instrument(skip(self))] - async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { - self.get_disks_by_key(object).transition_object(bucket, object, opts).await - } - - #[tracing::instrument(skip(self))] - async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { - self.get_disks_by_key(object).add_partial(bucket, object, version_id).await - } - - #[tracing::instrument(skip(self))] - async fn restore_transitioned_object(self: Arc, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { - self.get_disks_by_key(object) - .restore_transitioned_object(bucket, object, opts) - .await - } - #[tracing::instrument(skip(self))] async fn copy_object_part( &self, @@ -696,6 +676,20 @@ impl StorageAPI for Sets { .await } + async fn list_object_parts( + &self, + bucket: &str, + object: &str, + upload_id: &str, + part_number_marker: Option, + max_parts: usize, + opts: &ObjectOptions, + ) -> Result { + self.get_disks_by_key(object) + .list_object_parts(bucket, object, upload_id, part_number_marker, max_parts, opts) + .await + } + #[tracing::instrument(skip(self))] async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, opts: &ObjectOptions) -> Result<()> { self.get_disks_by_key(object) @@ -716,39 +710,10 @@ impl StorageAPI for Sets { .complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts) .await } +} - #[tracing::instrument(skip(self))] - async fn get_disks(&self, _pool_idx: usize, _set_idx: usize) -> Result>> { - unimplemented!() - } - - #[tracing::instrument(skip(self))] - fn set_drive_counts(&self) -> Vec { - unimplemented!() - } - - #[tracing::instrument(skip(self))] - async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { - self.get_disks_by_key(object).put_object_metadata(bucket, object, opts).await - } - - #[tracing::instrument(skip(self))] - async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { - self.get_disks_by_key(object).get_object_tags(bucket, object, opts).await - } - - #[tracing::instrument(level = "debug", skip(self))] - async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result { - self.get_disks_by_key(object) - .put_object_tags(bucket, object, tags, opts) - .await - } - - #[tracing::instrument(skip(self))] - async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { - self.get_disks_by_key(object).delete_object_tags(bucket, object, opts).await - } - +#[async_trait::async_trait] +impl HealOperations for Sets { #[tracing::instrument(skip(self))] async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { let (disks, _) = init_storage_disks_with_errors( @@ -857,16 +822,68 @@ impl StorageAPI for Sets { async fn check_abandoned_parts(&self, _bucket: &str, _object: &str, _opts: &HealOpts) -> Result<()> { unimplemented!() } +} - #[tracing::instrument(level = "debug", skip(self))] - async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { - let gor = self.get_object_reader(bucket, object, None, HeaderMap::new(), opts).await?; - let mut reader = gor.stream; +#[async_trait::async_trait] +impl StorageAPI for Sets { + #[tracing::instrument(skip(self))] + async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result { + self.disk_set[0].new_ns_lock(bucket, object).await + } + #[tracing::instrument(skip(self))] + async fn backend_info(&self) -> rustfs_madmin::BackendInfo { + unimplemented!() + } + #[tracing::instrument(skip(self))] + async fn storage_info(&self) -> rustfs_madmin::StorageInfo { + let mut futures = Vec::with_capacity(self.disk_set.len()); - // Stream data to sink instead of reading all into memory to prevent OOM - tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?; + for set in self.disk_set.iter() { + futures.push(set.storage_info()) + } - Ok(()) + let results = join_all(futures).await; + + let mut disks = Vec::new(); + + for res in results.into_iter() { + disks.extend_from_slice(&res.disks); + } + + rustfs_madmin::StorageInfo { + disks, + ..Default::default() + } + } + #[tracing::instrument(skip(self))] + async fn local_storage_info(&self) -> rustfs_madmin::StorageInfo { + let mut futures = Vec::with_capacity(self.disk_set.len()); + + for set in self.disk_set.iter() { + futures.push(set.local_storage_info()) + } + + let results = join_all(futures).await; + + let mut disks = Vec::new(); + + for res in results.into_iter() { + disks.extend_from_slice(&res.disks); + } + rustfs_madmin::StorageInfo { + disks, + ..Default::default() + } + } + + #[tracing::instrument(skip(self))] + async fn get_disks(&self, _pool_idx: usize, _set_idx: usize) -> Result>> { + unimplemented!() + } + + #[tracing::instrument(skip(self))] + fn set_drive_counts(&self) -> Vec { + unimplemented!() } } diff --git a/crates/ecstore/src/store.rs b/crates/ecstore/src/store.rs index ed8ad8301..c2d1c870b 100644 --- a/crates/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -58,9 +58,9 @@ use crate::{ rpc::S3PeerSys, sets::Sets, store_api::{ - BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec, - ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, - PutObjReader, StorageAPI, + BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, + HTTPRangeSpec, HealOperations, ListObjectsV2Info, ListOperations, MakeBucketOptions, MultipartOperations, + MultipartUploadResult, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, }, store_init, }; @@ -259,24 +259,7 @@ lazy_static! { } #[async_trait::async_trait] -impl StorageAPI for ECStore { - #[instrument(skip(self))] - async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result { - self.handle_new_ns_lock(bucket, object).await - } - #[instrument(skip(self))] - async fn backend_info(&self) -> rustfs_madmin::BackendInfo { - self.handle_backend_info().await - } - #[instrument(skip(self))] - async fn storage_info(&self) -> rustfs_madmin::StorageInfo { - self.handle_storage_info().await - } - #[instrument(skip(self))] - async fn local_storage_info(&self) -> rustfs_madmin::StorageInfo { - self.handle_local_storage_info().await - } - +impl BucketOperations for ECStore { #[instrument(skip(self))] async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> { self.handle_make_bucket(bucket, opts).await @@ -294,7 +277,91 @@ impl StorageAPI for ECStore { async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { self.handle_delete_bucket(bucket, opts).await } +} +#[async_trait::async_trait] +impl ObjectOperations for ECStore { + #[instrument(skip(self))] + async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { + self.handle_get_object_info(bucket, object, opts).await + } + + async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + self.handle_verify_object_integrity(bucket, object, opts).await + } + + // TODO: review + #[instrument(skip(self))] + async fn copy_object( + &self, + src_bucket: &str, + src_object: &str, + dst_bucket: &str, + dst_object: &str, + src_info: &mut ObjectInfo, + src_opts: &ObjectOptions, + dst_opts: &ObjectOptions, + ) -> Result { + self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts) + .await + } + + #[instrument(skip(self))] + async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> { + self.handle_delete_object_version(bucket, object, fi, force_del_marker).await + } + + #[instrument(skip(self))] + async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result { + self.handle_delete_object(bucket, object, opts).await + } + // TODO: review + #[instrument(skip(self))] + async fn delete_objects( + &self, + bucket: &str, + objects: Vec, + opts: ObjectOptions, + ) -> (Vec, Vec>) { + self.handle_delete_objects(bucket, objects, opts).await + } + + #[instrument(skip(self))] + async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { + self.handle_put_object_metadata(bucket, object, opts).await + } + #[instrument(skip(self))] + async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { + self.handle_get_object_tags(bucket, object, opts).await + } + + #[instrument(level = "debug", skip(self))] + async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result { + self.handle_put_object_tags(bucket, object, tags, opts).await + } + + #[instrument(skip(self))] + async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { + self.handle_delete_object_tags(bucket, object, opts).await + } + + #[instrument(skip(self))] + async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { + self.handle_add_partial(bucket, object, version_id).await + } + #[instrument(skip(self))] + async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + self.handle_transition_object(bucket, object, opts).await + } + + #[instrument(skip(self))] + async fn restore_transitioned_object(self: Arc, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { + self.handle_restore_transitioned_object(bucket, object, opts).await + } +} + +#[async_trait::async_trait] +impl ListOperations for ECStore { // @continuation_token marker // @start_after as marker when continuation_token empty // @delimiter default="/", empty when recursive @@ -348,56 +415,10 @@ impl StorageAPI for ECStore { ) -> Result<()> { self.handle_walk(rx, bucket, prefix, result, opts).await } +} - #[instrument(skip(self))] - async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { - self.handle_get_object_info(bucket, object, opts).await - } - - // TODO: review - #[instrument(skip(self))] - async fn copy_object( - &self, - src_bucket: &str, - src_object: &str, - dst_bucket: &str, - dst_object: &str, - src_info: &mut ObjectInfo, - src_opts: &ObjectOptions, - dst_opts: &ObjectOptions, - ) -> Result { - self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts) - .await - } - #[instrument(skip(self))] - async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result { - self.handle_delete_object(bucket, object, opts).await - } - // TODO: review - #[instrument(skip(self))] - async fn delete_objects( - &self, - bucket: &str, - objects: Vec, - opts: ObjectOptions, - ) -> (Vec, Vec>) { - self.handle_delete_objects(bucket, objects, opts).await - } - - #[instrument(skip(self))] - async fn list_object_parts( - &self, - bucket: &str, - object: &str, - upload_id: &str, - part_number_marker: Option, - max_parts: usize, - opts: &ObjectOptions, - ) -> Result { - self.handle_list_object_parts(bucket, object, upload_id, part_number_marker, max_parts, opts) - .await - } - +#[async_trait::async_trait] +impl MultipartOperations for ECStore { #[instrument(skip(self))] async fn list_multipart_uploads( &self, @@ -417,20 +438,6 @@ impl StorageAPI for ECStore { self.handle_new_multipart_upload(bucket, object, opts).await } - #[instrument(skip(self))] - async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> { - self.handle_add_partial(bucket, object, version_id).await - } - #[instrument(skip(self))] - async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { - self.handle_transition_object(bucket, object, opts).await - } - - #[instrument(skip(self))] - async fn restore_transitioned_object(self: Arc, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { - self.handle_restore_transitioned_object(bucket, object, opts).await - } - #[instrument(skip(self))] async fn copy_object_part( &self, @@ -485,6 +492,21 @@ impl StorageAPI for ECStore { ) -> Result { self.handle_get_multipart_info(bucket, object, upload_id, opts).await } + + #[instrument(skip(self))] + async fn list_object_parts( + &self, + bucket: &str, + object: &str, + upload_id: &str, + part_number_marker: Option, + max_parts: usize, + opts: &ObjectOptions, + ) -> Result { + self.handle_list_object_parts(bucket, object, upload_id, part_number_marker, max_parts, opts) + .await + } + #[instrument(skip(self))] async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str, opts: &ObjectOptions) -> Result<()> { self.handle_abort_multipart_upload(bucket, object, upload_id, opts).await @@ -502,40 +524,10 @@ impl StorageAPI for ECStore { self.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts) .await } +} - #[instrument(skip(self))] - async fn get_disks(&self, pool_idx: usize, set_idx: usize) -> Result>> { - self.handle_get_disks(pool_idx, set_idx).await - } - - #[instrument(skip(self))] - fn set_drive_counts(&self) -> Vec { - self.handle_set_drive_counts() - } - #[instrument(skip(self))] - async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { - self.handle_put_object_metadata(bucket, object, opts).await - } - #[instrument(skip(self))] - async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { - self.handle_get_object_tags(bucket, object, opts).await - } - - #[instrument(level = "debug", skip(self))] - async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result { - self.handle_put_object_tags(bucket, object, tags, opts).await - } - - #[instrument(skip(self))] - async fn delete_object_version(&self, bucket: &str, object: &str, fi: &FileInfo, force_del_marker: bool) -> Result<()> { - self.handle_delete_object_version(bucket, object, fi, force_del_marker).await - } - - #[instrument(skip(self))] - async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { - self.handle_delete_object_tags(bucket, object, opts).await - } - +#[async_trait::async_trait] +impl HealOperations for ECStore { #[instrument(skip(self))] async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)> { self.handle_heal_format(dry_run).await @@ -565,9 +557,35 @@ impl StorageAPI for ECStore { async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> { self.handle_check_abandoned_parts(bucket, object, opts).await } +} - async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> { - self.handle_verify_object_integrity(bucket, object, opts).await +#[async_trait::async_trait] +impl StorageAPI for ECStore { + #[instrument(skip(self))] + async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result { + self.handle_new_ns_lock(bucket, object).await + } + #[instrument(skip(self))] + async fn backend_info(&self) -> rustfs_madmin::BackendInfo { + self.handle_backend_info().await + } + #[instrument(skip(self))] + async fn storage_info(&self) -> rustfs_madmin::StorageInfo { + self.handle_storage_info().await + } + #[instrument(skip(self))] + async fn local_storage_info(&self) -> rustfs_madmin::StorageInfo { + self.handle_local_storage_info().await + } + + #[instrument(skip(self))] + async fn get_disks(&self, pool_idx: usize, set_idx: usize) -> Result>> { + self.handle_get_disks(pool_idx, set_idx).await + } + + #[instrument(skip(self))] + fn set_drive_counts(&self) -> Vec { + self.handle_set_drive_counts() } } diff --git a/crates/ecstore/src/store_api/traits.rs b/crates/ecstore/src/store_api/traits.rs index 84fd431f6..facabeae3 100644 --- a/crates/ecstore/src/store_api/traits.rs +++ b/crates/ecstore/src/store_api/traits.rs @@ -2,7 +2,6 @@ use super::*; #[async_trait::async_trait] pub trait ObjectIO: Send + Sync + Debug + 'static { - // GetObjectNInfo FIXME: async fn get_object_reader( &self, bucket: &str, @@ -11,55 +10,23 @@ pub trait ObjectIO: Send + Sync + Debug + 'static { h: HeaderMap, opts: &ObjectOptions, ) -> Result; - // PutObject + async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result; } +/// Bucket-level storage operations. #[async_trait::async_trait] -#[allow(clippy::too_many_arguments)] -pub trait StorageAPI: ObjectIO + Debug { - async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result; - - async fn backend_info(&self) -> rustfs_madmin::BackendInfo; - async fn storage_info(&self) -> rustfs_madmin::StorageInfo; - async fn local_storage_info(&self) -> rustfs_madmin::StorageInfo; - +pub trait BucketOperations: Send + Sync + Debug { async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()>; async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result; async fn list_bucket(&self, opts: &BucketOptions) -> Result>; async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()>; - // ListObjects TODO: FIXME: - async fn list_objects_v2( - self: Arc, - bucket: &str, - prefix: &str, - continuation_token: Option, - delimiter: Option, - max_keys: i32, - fetch_owner: bool, - start_after: Option, - incl_deleted: bool, - ) -> Result; - // ListObjectVersions TODO: FIXME: - async fn list_object_versions( - self: Arc, - bucket: &str, - prefix: &str, - marker: Option, - version_marker: Option, - delimiter: Option, - max_keys: i32, - ) -> Result; - - async fn walk( - self: Arc, - rx: CancellationToken, - bucket: &str, - prefix: &str, - result: tokio::sync::mpsc::Sender, - opts: WalkOptions, - ) -> Result<()>; +} +/// Object-level storage operations (beyond basic I/O in ObjectIO). +#[async_trait::async_trait] +#[allow(clippy::too_many_arguments)] +pub trait ObjectOperations: Send + Sync + Debug { async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>; async fn copy_object( @@ -80,10 +47,55 @@ pub trait StorageAPI: ObjectIO + Debug { objects: Vec, opts: ObjectOptions, ) -> (Vec, Vec>); + async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; + async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; + async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result; + async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; + async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()>; + async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>; + async fn restore_transitioned_object(self: Arc, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>; +} - // TransitionObject TODO: - // RestoreTransitionedObject TODO: +/// Listing and walking operations. +#[async_trait::async_trait] +#[allow(clippy::too_many_arguments)] +pub trait ListOperations: Send + Sync + Debug { + async fn list_objects_v2( + self: Arc, + bucket: &str, + prefix: &str, + continuation_token: Option, + delimiter: Option, + max_keys: i32, + fetch_owner: bool, + start_after: Option, + incl_deleted: bool, + ) -> Result; + async fn list_object_versions( + self: Arc, + bucket: &str, + prefix: &str, + marker: Option, + version_marker: Option, + delimiter: Option, + max_keys: i32, + ) -> Result; + + async fn walk( + self: Arc, + rx: CancellationToken, + bucket: &str, + prefix: &str, + result: tokio::sync::mpsc::Sender, + opts: WalkOptions, + ) -> Result<()>; +} + +/// Multipart upload operations. +#[async_trait::async_trait] +#[allow(clippy::too_many_arguments)] +pub trait MultipartOperations: Send + Sync + Debug { async fn list_multipart_uploads( &self, bucket: &str, @@ -142,19 +154,11 @@ pub trait StorageAPI: ObjectIO + Debug { uploaded_parts: Vec, opts: &ObjectOptions, ) -> Result; - async fn get_disks(&self, pool_idx: usize, set_idx: usize) -> Result>>; - fn set_drive_counts(&self) -> Vec; - - // Health TODO: - async fn put_object_metadata(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; - // DecomTieredObject - async fn get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; - async fn add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()>; - async fn transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>; - async fn restore_transitioned_object(self: Arc, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()>; - async fn put_object_tags(&self, bucket: &str, object: &str, tags: &str, opts: &ObjectOptions) -> Result; - async fn delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; +} +/// Healing and repair operations. +#[async_trait::async_trait] +pub trait HealOperations: Send + Sync + Debug { async fn heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option)>; async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result; async fn heal_object( @@ -164,8 +168,25 @@ pub trait StorageAPI: ObjectIO + Debug { version_id: &str, opts: &HealOpts, ) -> Result<(HealResultItem, Option)>; - // async fn heal_objects(&self, bucket: &str, prefix: &str, opts: &HealOpts, hs: Arc, is_meta: bool) - // -> Result<()>; async fn get_pool_and_set(&self, id: &str) -> Result<(Option, Option, Option)>; async fn check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()>; } + +/// Unified storage API combining all operation groups. +/// +/// Consumers can depend on specific sub-traits (e.g., `BucketOperations`) +/// when they don't need the full API surface. +#[async_trait::async_trait] +#[allow(clippy::too_many_arguments)] +pub trait StorageAPI: + ObjectIO + BucketOperations + ObjectOperations + ListOperations + MultipartOperations + HealOperations + Debug +{ + async fn new_ns_lock(&self, bucket: &str, object: &str) -> Result; + + async fn backend_info(&self) -> rustfs_madmin::BackendInfo; + async fn storage_info(&self) -> rustfs_madmin::StorageInfo; + async fn local_storage_info(&self) -> rustfs_madmin::StorageInfo; + + async fn get_disks(&self, pool_idx: usize, set_idx: usize) -> Result>>; + fn set_drive_counts(&self) -> Vec; +} diff --git a/crates/ecstore/src/store_list_objects.rs b/crates/ecstore/src/store_list_objects.rs index 892587e3f..1e06f5235 100644 --- a/crates/ecstore/src/store_list_objects.rs +++ b/crates/ecstore/src/store_list_objects.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::StorageAPI; use crate::bucket::metadata_sys::get_versioning_config; use crate::bucket::utils::check_list_objs_args; use crate::bucket::versioning::VersioningApi; @@ -24,7 +23,8 @@ use crate::error::{ }; use crate::set_disk::SetDisks; use crate::store_api::{ - ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectInfoOrErr, ObjectOptions, WalkOptions, WalkVersionsSortOrder, + ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectInfoOrErr, ObjectOperations, ObjectOptions, WalkOptions, + WalkVersionsSortOrder, }; use crate::store_utils::is_reserved_or_invalid_bucket; use crate::{store::ECStore, store_api::ListObjectsV2Info}; diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index 369c475cd..33f6210bc 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -18,7 +18,7 @@ use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_ecstore::{ disk::{DiskStore, endpoint::Endpoint}, store::ECStore, - store_api::{BucketInfo, ObjectIO, StorageAPI}, + store_api::{BucketInfo, BucketOperations, HealOperations, ListOperations, ObjectIO, ObjectOperations, StorageAPI}, }; use rustfs_madmin::heal_commands::HealResultItem; use std::sync::Arc; diff --git a/crates/heal/tests/heal_integration_test.rs b/crates/heal/tests/heal_integration_test.rs index 5407ae01a..509c5cd93 100644 --- a/crates/heal/tests/heal_integration_test.rs +++ b/crates/heal/tests/heal_integration_test.rs @@ -18,7 +18,7 @@ use rustfs_ecstore::{ disk::endpoint::Endpoint, endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, store::ECStore, - store_api::{ObjectIO, ObjectOptions, PutObjReader, StorageAPI}, + store_api::{BucketOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}, }; use rustfs_heal::heal::{ manager::{HealConfig, HealManager}, diff --git a/crates/iam/src/store/object.rs b/crates/iam/src/store/object.rs index 930a8743d..4f6e16119 100644 --- a/crates/iam/src/store/object.rs +++ b/crates/iam/src/store/object.rs @@ -21,8 +21,7 @@ use crate::{ }; use futures::future::join_all; use rustfs_credentials::get_global_action_cred; -use rustfs_ecstore::StorageAPI as _; -use rustfs_ecstore::store_api::{ObjectInfoOrErr, WalkOptions}; +use rustfs_ecstore::store_api::{ListOperations as _, ObjectInfoOrErr, WalkOptions}; use rustfs_ecstore::{ config::{ RUSTFS_CONFIG_PREFIX, diff --git a/crates/metrics/src/collectors/global.rs b/crates/metrics/src/collectors/global.rs index 9e1011f21..21509a8f2 100644 --- a/crates/metrics/src/collectors/global.rs +++ b/crates/metrics/src/collectors/global.rs @@ -25,7 +25,7 @@ use crate::format::report_metrics; use rustfs_ecstore::bucket::metadata_sys::get_quota_config; use rustfs_ecstore::data_usage::load_data_usage_from_backend; use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free}; -use rustfs_ecstore::store_api::BucketOptions; +use rustfs_ecstore::store_api::{BucketOperations, BucketOptions}; use rustfs_ecstore::{StorageAPI, new_object_layer_fn}; use rustfs_utils::get_env_opt_u64; use std::sync::OnceLock; diff --git a/crates/policy/src/policy/variables.rs b/crates/policy/src/policy/variables.rs index db35663e4..4c02ae089 100644 --- a/crates/policy/src/policy/variables.rs +++ b/crates/policy/src/policy/variables.rs @@ -161,7 +161,7 @@ impl VariableResolver { fn resolve_userid(&self) -> Option { self.get_claim_as_strings("sub") .or_else(|| self.get_claim_as_strings("parent")) - .and_then(|mut vec| vec.pop()) // 取第一个值，保持原有逻辑 + .and_then(|mut vec| vec.pop()) } fn resolve_principal_type(&self) -> String { diff --git a/crates/s3select-api/src/object_store.rs b/crates/s3select-api/src/object_store.rs index 56b6ebb76..ebcd72292 100644 --- a/crates/s3select-api/src/object_store.rs +++ b/crates/s3select-api/src/object_store.rs @@ -25,11 +25,11 @@ use object_store::{ }; use pin_project_lite::pin_project; use rustfs_common::DEFAULT_DELIMITER; -use rustfs_ecstore::StorageAPI; use rustfs_ecstore::new_object_layer_fn; use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; use rustfs_ecstore::store::ECStore; use rustfs_ecstore::store_api::ObjectIO; +use rustfs_ecstore::store_api::ObjectOperations; use rustfs_ecstore::store_api::ObjectOptions; use s3s::S3Result; use s3s::dto::SelectObjectContentInput; diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index f5e6c9390..14aa40767 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -34,7 +34,7 @@ use rustfs_ecstore::error::{Error, StorageError}; use rustfs_ecstore::global::GLOBAL_TierConfigMgr; use rustfs_ecstore::new_object_layer_fn; use rustfs_ecstore::set_disk::SetDisks; -use rustfs_ecstore::store_api::{BucketInfo, BucketOptions, ObjectInfo}; +use rustfs_ecstore::store_api::{BucketInfo, BucketOperations, BucketOptions, ObjectInfo}; use rustfs_ecstore::{StorageAPI, error::Result, store::ECStore}; use rustfs_filemeta::FileMeta; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; diff --git a/crates/scanner/tests/lifecycle_integration_test.rs b/crates/scanner/tests/lifecycle_integration_test.rs index a74678804..31cfba1d2 100644 --- a/crates/scanner/tests/lifecycle_integration_test.rs +++ b/crates/scanner/tests/lifecycle_integration_test.rs @@ -19,7 +19,7 @@ use rustfs_ecstore::{ endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, global::GLOBAL_TierConfigMgr, store::ECStore, - store_api::{MakeBucketOptions, ObjectIO, ObjectOptions, PutObjReader, StorageAPI}, + store_api::{BucketOperations, MakeBucketOptions, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader}, tier::tier_config::{TierConfig, TierMinIO, TierType}, }; use rustfs_scanner::scanner::init_data_scanner; diff --git a/rustfs/src/admin/handlers/account_info.rs b/rustfs/src/admin/handlers/account_info.rs index 967f1b91b..756f04e1c 100644 --- a/rustfs/src/admin/handlers/account_info.rs +++ b/rustfs/src/admin/handlers/account_info.rs @@ -21,7 +21,7 @@ use matchit::Params; use rustfs_credentials::get_global_action_cred; use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys; use rustfs_ecstore::new_object_layer_fn; -use rustfs_ecstore::store_api::{BucketOptions, StorageAPI}; +use rustfs_ecstore::store_api::{BucketOperations, BucketOptions, StorageAPI}; use rustfs_iam::store::MappedPolicy; use rustfs_policy::policy::BucketPolicy; use rustfs_policy::policy::default::DEFAULT_POLICIES; diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index 94e75cb0e..10ac47356 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -25,7 +25,10 @@ use hyper::Method; use matchit::Params; use rustfs_config::MAX_BUCKET_METADATA_IMPORT_SIZE; use rustfs_ecstore::{ - StorageAPI, + bucket::utils::{deserialize, serialize}, + store_api::MakeBucketOptions, +}; +use rustfs_ecstore::{ bucket::{ metadata::{ BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, @@ -38,11 +41,7 @@ use rustfs_ecstore::{ }, error::StorageError, new_object_layer_fn, - store_api::BucketOptions, -}; -use rustfs_ecstore::{ - bucket::utils::{deserialize, serialize}, - store_api::MakeBucketOptions, + store_api::{BucketOperations, BucketOptions}, }; use rustfs_policy::policy::{ BucketPolicy, diff --git a/rustfs/src/admin/handlers/rebalance.rs b/rustfs/src/admin/handlers/rebalance.rs index 9be46f8bc..5a97ac73f 100644 --- a/rustfs/src/admin/handlers/rebalance.rs +++ b/rustfs/src/admin/handlers/rebalance.rs @@ -30,6 +30,7 @@ use rustfs_ecstore::{ new_object_layer_fn, notification_sys::get_global_notification_sys, rebalance::{DiskStat, RebalSaveOpt}, + store_api::BucketOperations, store_api::BucketOptions, }; use rustfs_policy::policy::action::{Action, AdminAction}; diff --git a/rustfs/src/admin/handlers/replication.rs b/rustfs/src/admin/handlers/replication.rs index b07145267..a29f694d3 100644 --- a/rustfs/src/admin/handlers/replication.rs +++ b/rustfs/src/admin/handlers/replication.rs @@ -27,7 +27,7 @@ use rustfs_ecstore::bucket::metadata_sys; use rustfs_ecstore::bucket::target::BucketTarget; use rustfs_ecstore::global::global_rustfs_port; use rustfs_ecstore::new_object_layer_fn; -use rustfs_ecstore::store_api::{BucketOptions, StorageAPI}; +use rustfs_ecstore::store_api::{BucketOperations, BucketOptions}; use rustfs_policy::policy::action::{Action, AdminAction}; use s3s::header::CONTENT_TYPE; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; diff --git a/rustfs/src/app/admin_usecase.rs b/rustfs/src/app/admin_usecase.rs index a24d21f0a..ddcbbc980 100644 --- a/rustfs/src/app/admin_usecase.rs +++ b/rustfs/src/app/admin_usecase.rs @@ -13,7 +13,6 @@ // limitations under the License. //! Admin application use-case contracts. -#![allow(dead_code)] use crate::app::context::{AppContext, get_global_app_context}; use crate::error::ApiError; @@ -58,31 +57,13 @@ pub struct QueryPoolStatusRequest { pub by_id: bool, } -#[async_trait::async_trait] -pub trait AdminUsecase: Send + Sync { - async fn query_server_info(&self, req: QueryServerInfoRequest) -> AdminUsecaseResult; - - async fn query_storage_info(&self) -> AdminUsecaseResult; - - async fn query_data_usage_info(&self) -> AdminUsecaseResult; - - async fn list_pool_statuses(&self) -> AdminUsecaseResult>; - - async fn query_pool_status(&self, req: QueryPoolStatusRequest) -> AdminUsecaseResult; - - fn collect_dependency_readiness(&self) -> DependencyReadiness; -} - #[derive(Clone, Default)] pub struct DefaultAdminUsecase { context: Option>, } impl DefaultAdminUsecase { - pub fn new(context: Arc) -> Self { - Self { context: Some(context) } - } - + #[cfg(test)] pub fn without_context() -> Self { Self { context: None } } @@ -93,10 +74,6 @@ impl DefaultAdminUsecase { } } - pub fn context(&self) -> Option> { - self.context.clone() - } - fn endpoints(&self) -> Option { self.context.as_ref().and_then(|context| context.endpoints().handle()) } @@ -292,33 +269,6 @@ impl DefaultAdminUsecase { } } -#[async_trait::async_trait] -impl AdminUsecase for DefaultAdminUsecase { - async fn query_server_info(&self, req: QueryServerInfoRequest) -> AdminUsecaseResult { - self.execute_query_server_info(req).await - } - - async fn query_storage_info(&self) -> AdminUsecaseResult { - self.execute_query_storage_info().await - } - - async fn query_data_usage_info(&self) -> AdminUsecaseResult { - self.execute_query_data_usage_info().await - } - - async fn list_pool_statuses(&self) -> AdminUsecaseResult> { - self.execute_list_pool_statuses().await - } - - async fn query_pool_status(&self, req: QueryPoolStatusRequest) -> AdminUsecaseResult { - self.execute_query_pool_status(req).await - } - - fn collect_dependency_readiness(&self) -> DependencyReadiness { - self.execute_collect_dependency_readiness() - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 337f7eb7f..30a4bdafb 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -13,7 +13,6 @@ // limitations under the License. //! Bucket application use-case contracts. -#![allow(dead_code)] use crate::app::context::{AppContext, default_notify_interface, get_global_app_context}; use crate::auth::get_condition_values; @@ -46,7 +45,7 @@ use rustfs_ecstore::bucket::{ use rustfs_ecstore::client::object_api_utils::to_s3s_etag; use rustfs_ecstore::error::StorageError; use rustfs_ecstore::new_object_layer_fn; -use rustfs_ecstore::store_api::{BucketOptions, DeleteBucketOptions, MakeBucketOptions, StorageAPI}; +use rustfs_ecstore::store_api::{BucketOperations, BucketOptions, DeleteBucketOptions, ListOperations, MakeBucketOptions}; use rustfs_policy::policy::{ action::{Action, S3Action}, {BucketPolicy, BucketPolicyArgs, Effect, Validator}, @@ -65,8 +64,6 @@ use std::{fmt::Display, sync::Arc}; use tracing::{debug, error, info, instrument, warn}; use urlencoding::encode; -pub type BucketUsecaseResult = Result; - fn serialize_config(value: &T) -> S3Result> { serialize(value).map_err(to_internal_error) } @@ -75,72 +72,13 @@ fn to_internal_error(err: impl Display) -> S3Error { S3Error::with_message(S3ErrorCode::InternalError, format!("{err}")) } +fn default_region() -> Region { + // RUSTFS_REGION is a compile-time constant ("us-east-1") guaranteed to be valid. + Region::new(RUSTFS_REGION.into()).expect("RUSTFS_REGION constant must be a valid region") +} + fn resolve_notification_region(global_region: Option, request_region: Option) -> Region { - global_region.unwrap_or_else(|| request_region.unwrap_or_else(|| Region::new(RUSTFS_REGION.into()).expect("valid region"))) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CreateBucketRequest { - pub bucket: String, - pub object_lock_enabled: Option, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct CreateBucketResponse; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DeleteBucketRequest { - pub bucket: String, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct DeleteBucketResponse; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HeadBucketRequest { - pub bucket: String, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct HeadBucketResponse; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ListObjectsV2Request { - pub bucket: String, - pub prefix: Option, - pub delimiter: Option, - pub continuation_token: Option, - pub max_keys: Option, - pub fetch_owner: Option, - pub start_after: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ListObjectsV2Item { - pub key: String, - pub etag: Option, - pub size: i64, - pub version_id: Option, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct ListObjectsV2Response { - pub objects: Vec, - pub common_prefixes: Vec, - pub key_count: i32, - pub is_truncated: bool, - pub next_continuation_token: Option, -} - -#[async_trait::async_trait] -pub trait BucketUsecase: Send + Sync { - async fn create_bucket(&self, req: CreateBucketRequest) -> BucketUsecaseResult; - - async fn delete_bucket(&self, req: DeleteBucketRequest) -> BucketUsecaseResult; - - async fn head_bucket(&self, req: HeadBucketRequest) -> BucketUsecaseResult; - - async fn list_objects_v2(&self, req: ListObjectsV2Request) -> BucketUsecaseResult; + global_region.or(request_region).unwrap_or_else(default_region) } #[derive(Clone, Default)] @@ -149,10 +87,7 @@ pub struct DefaultBucketUsecase { } impl DefaultBucketUsecase { - pub fn new(context: Arc) -> Self { - Self { context: Some(context) } - } - + #[cfg(test)] pub fn without_context() -> Self { Self { context: None } } @@ -163,10 +98,6 @@ impl DefaultBucketUsecase { } } - pub fn context(&self) -> Option> { - self.context.clone() - } - fn global_region(&self) -> Option { self.context.as_ref().and_then(|context| context.region().get()) } @@ -466,7 +397,10 @@ impl DefaultBucketUsecase { list_bucket_infos = futures::stream::iter(list_bucket_infos) .filter_map(|info| async { let mut req_clone = req.clone(); - let req_info = req_clone.extensions.get_mut::().expect("ReqInfo not found"); + let Some(req_info) = req_clone.extensions.get_mut::() else { + debug!(bucket = %info.name, "ReqInfo missing in extensions, skipping bucket authorization"); + return None; + }; req_info.bucket = Some(info.name.clone()); if authorize_request(&mut req_clone, Action::S3Action(S3Action::ListBucketAction)) @@ -1750,37 +1684,6 @@ impl DefaultBucketUsecase { } } -#[async_trait::async_trait] -impl BucketUsecase for DefaultBucketUsecase { - async fn create_bucket(&self, req: CreateBucketRequest) -> BucketUsecaseResult { - let _ = req; - Err(ApiError::from(StorageError::other( - "DefaultBucketUsecase::create_bucket DTO path is not implemented yet", - ))) - } - - async fn delete_bucket(&self, req: DeleteBucketRequest) -> BucketUsecaseResult { - let _ = req; - Err(ApiError::from(StorageError::other( - "DefaultBucketUsecase::delete_bucket DTO path is not implemented yet", - ))) - } - - async fn head_bucket(&self, req: HeadBucketRequest) -> BucketUsecaseResult { - let _ = req; - Err(ApiError::from(StorageError::other( - "DefaultBucketUsecase::head_bucket DTO path is not implemented yet", - ))) - } - - async fn list_objects_v2(&self, req: ListObjectsV2Request) -> BucketUsecaseResult { - let _ = req; - Err(ApiError::from(StorageError::other( - "DefaultBucketUsecase::list_objects_v2 DTO path is not implemented yet", - ))) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/rustfs/src/app/context.rs b/rustfs/src/app/context.rs index c42304090..287d0df52 100644 --- a/rustfs/src/app/context.rs +++ b/rustfs/src/app/context.rs @@ -15,7 +15,6 @@ //! Application-layer dependency context. //! This module introduces explicit dependency injection entry points //! for storage, IAM, and KMS handles. -#![allow(dead_code)] use crate::config::workload_profiles::{RustFSBufferConfig, get_global_buffer_config}; use async_trait::async_trait; @@ -34,11 +33,13 @@ use tokio::sync::RwLock; /// IAM interface for application-layer use-cases. pub trait IamInterface: Send + Sync { + #[allow(dead_code)] fn handle(&self) -> Arc>; fn is_ready(&self) -> bool; } /// KMS interface for application-layer use-cases. +#[allow(dead_code)] pub trait KmsInterface: Send + Sync { fn handle(&self) -> Arc; } @@ -95,6 +96,7 @@ pub trait BufferConfigInterface: Send + Sync { /// Default IAM interface adapter. pub struct IamHandle { + #[allow(dead_code)] iam: Arc>, } @@ -115,6 +117,7 @@ impl IamInterface for IamHandle { } /// Default KMS interface adapter. +#[allow(dead_code)] pub struct KmsHandle { kms: Arc, } @@ -230,6 +233,7 @@ impl BufferConfigInterface for BufferConfigHandle { pub struct AppContext { object_store: Arc, iam: Arc, + #[allow(dead_code)] kms: Arc, kms_runtime: Arc, notify: Arc, @@ -274,6 +278,7 @@ impl AppContext { self.iam.clone() } + #[allow(dead_code)] pub fn kms(&self) -> Arc { self.kms.clone() } diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index 824a39441..72e3fee81 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -13,7 +13,6 @@ // limitations under the License. //! Multipart application use-case contracts. -#![allow(dead_code)] use crate::app::context::{AppContext, get_global_app_context}; use crate::error::ApiError; @@ -28,7 +27,6 @@ use crate::storage::*; use bytes::Bytes; use futures::StreamExt; use rustfs_config::RUSTFS_REGION; -use rustfs_ecstore::StorageAPI; use rustfs_ecstore::bucket::quota::checker::QuotaChecker; use rustfs_ecstore::bucket::{ metadata_sys, @@ -41,6 +39,7 @@ use rustfs_ecstore::error::{StorageError, is_err_object_not_found, is_err_versio use rustfs_ecstore::new_object_layer_fn; use rustfs_ecstore::set_disk::{MAX_PARTS_COUNT, is_valid_storage_class}; use rustfs_ecstore::store_api::{CompletePart, MultipartUploadResult, ObjectIO, ObjectOptions, PutObjReader}; +use rustfs_ecstore::store_api::{MultipartOperations, ObjectOperations}; use rustfs_filemeta::{ReplicationStatusType, ReplicationType}; use rustfs_rio::{CompressReader, HashReader, Reader, WarpReader}; use rustfs_targets::EventName; @@ -50,6 +49,7 @@ use rustfs_utils::http::{ headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX_LOWER}, }; use s3s::dto::*; +use s3s::region::Region; use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; use std::collections::HashMap; use std::str::FromStr; @@ -58,95 +58,13 @@ use tokio::sync::RwLock; use tokio_util::io::StreamReader; use tracing::{info, instrument, warn}; -pub type MultipartUsecaseResult = Result; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CreateMultipartUploadRequest { - pub bucket: String, - pub key: String, - pub metadata: HashMap, - pub content_type: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CreateMultipartUploadResponse { - pub upload_id: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UploadPartRequest { - pub bucket: String, - pub key: String, - pub upload_id: String, - pub part_number: i32, - pub content_length: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct UploadPartResponse { - pub etag: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CompleteMultipartUploadPart { - pub part_number: i32, - pub etag: String, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CompleteMultipartUploadRequest { - pub bucket: String, - pub key: String, - pub upload_id: String, - pub parts: Vec, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct CompleteMultipartUploadResponse { - pub etag: Option, - pub version_id: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AbortMultipartUploadRequest { - pub bucket: String, - pub key: String, - pub upload_id: String, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct AbortMultipartUploadResponse; - -#[async_trait::async_trait] -pub trait MultipartUsecase: Send + Sync { - async fn create_multipart_upload( - &self, - req: CreateMultipartUploadRequest, - ) -> MultipartUsecaseResult; - - async fn upload_part(&self, req: UploadPartRequest) -> MultipartUsecaseResult; - - async fn complete_multipart_upload( - &self, - req: CompleteMultipartUploadRequest, - ) -> MultipartUsecaseResult; - - async fn abort_multipart_upload( - &self, - req: AbortMultipartUploadRequest, - ) -> MultipartUsecaseResult; -} - #[derive(Clone, Default)] pub struct DefaultMultipartUsecase { context: Option>, } impl DefaultMultipartUsecase { - pub fn new(context: Arc) -> Self { - Self { context: Some(context) } - } - + #[cfg(test)] pub fn without_context() -> Self { Self { context: None } } @@ -157,10 +75,6 @@ impl DefaultMultipartUsecase { } } - pub fn context(&self) -> Option> { - self.context.clone() - } - fn bucket_metadata_sys(&self) -> Option>> { self.context.as_ref().and_then(|context| context.bucket_metadata().handle()) } @@ -423,7 +337,9 @@ impl DefaultMultipartUsecase { } } - let region = self.global_region().unwrap_or_else(|| RUSTFS_REGION.parse().unwrap()); + let region = self + .global_region() + .unwrap_or_else(|| Region::new(RUSTFS_REGION.into()).expect("RUSTFS_REGION constant must be a valid region")); let output = CompleteMultipartUploadOutput { bucket: Some(bucket.clone()), key: Some(key.clone()), @@ -1181,46 +1097,6 @@ impl DefaultMultipartUsecase { } } -#[async_trait::async_trait] -impl MultipartUsecase for DefaultMultipartUsecase { - async fn create_multipart_upload( - &self, - req: CreateMultipartUploadRequest, - ) -> MultipartUsecaseResult { - let _ = req; - Err(ApiError::from(StorageError::other( - "DefaultMultipartUsecase::create_multipart_upload is not implemented yet", - ))) - } - - async fn upload_part(&self, req: UploadPartRequest) -> MultipartUsecaseResult { - let _ = req; - Err(ApiError::from(StorageError::other( - "DefaultMultipartUsecase::upload_part is not implemented yet", - ))) - } - - async fn complete_multipart_upload( - &self, - req: CompleteMultipartUploadRequest, - ) -> MultipartUsecaseResult { - let _ = req; - Err(ApiError::from(StorageError::other( - "DefaultMultipartUsecase::complete_multipart_upload is not implemented yet", - ))) - } - - async fn abort_multipart_upload( - &self, - req: AbortMultipartUploadRequest, - ) -> MultipartUsecaseResult { - let _ = req; - Err(ApiError::from(StorageError::other( - "DefaultMultipartUsecase::abort_multipart_upload is not implemented yet", - ))) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 9885f222d..adba060e2 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -13,12 +13,11 @@ // limitations under the License. //! Object application use-case contracts. -#![allow(dead_code)] use crate::app::context::{AppContext, default_notify_interface, get_global_app_context}; use crate::config::workload_profiles::RustFSBufferConfig; use crate::error::ApiError; -use crate::storage::access::{ReqInfo, authorize_request, has_bypass_governance_header}; +use crate::storage::access::{ReqInfo, authorize_request, has_bypass_governance_header, req_info_mut}; use crate::storage::concurrency::{ CachedGetObject, ConcurrencyManager, GetObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, }; @@ -38,7 +37,6 @@ use datafusion::arrow::{ use futures::StreamExt; use http::{HeaderMap, HeaderValue, StatusCode}; use metrics::{counter, histogram}; -use rustfs_ecstore::StorageAPI; use rustfs_ecstore::bucket::quota::checker::QuotaChecker; use rustfs_ecstore::bucket::{ lifecycle::{ @@ -65,7 +63,8 @@ use rustfs_ecstore::error::{StorageError, is_err_bucket_not_found, is_err_object use rustfs_ecstore::new_object_layer_fn; use rustfs_ecstore::set_disk::is_valid_storage_class; use rustfs_ecstore::store_api::{ - BucketOptions, HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOptions, ObjectToDelete, PutObjReader, + BucketOperations, BucketOptions, HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, ObjectToDelete, + PutObjReader, }; use rustfs_filemeta::{ REPLICATE_INCOMING_DELETE, ReplicationStatusType, ReplicationType, RestoreStatusOps, VersionPurgeStatusType, @@ -112,7 +111,49 @@ use tokio_util::io::{ReaderStream, StreamReader}; use tracing::{debug, error, info, instrument, warn}; use uuid::Uuid; -pub type ObjectUsecaseResult = Result; +/// Extract trailing-header checksum values, overriding the corresponding input fields. +fn apply_trailing_checksums( + algorithm: Option<&str>, + trailing_headers: &Option, + checksums: &mut PutObjectChecksums, +) { + let Some(alg) = algorithm else { return }; + let Some(checksum_str) = trailing_headers.as_ref().and_then(|trailer| { + let key = match alg { + ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(), + ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(), + ChecksumAlgorithm::SHA1 => rustfs_rio::ChecksumType::SHA1.key(), + ChecksumAlgorithm::SHA256 => rustfs_rio::ChecksumType::SHA256.key(), + ChecksumAlgorithm::CRC64NVME => rustfs_rio::ChecksumType::CRC64_NVME.key(), + _ => return None, + }; + trailer.read(|headers| { + headers + .get(key.unwrap_or_default()) + .and_then(|value| value.to_str().ok().map(|s| s.to_string())) + }) + }) else { + return; + }; + + match alg { + ChecksumAlgorithm::CRC32 => checksums.crc32 = checksum_str, + ChecksumAlgorithm::CRC32C => checksums.crc32c = checksum_str, + ChecksumAlgorithm::SHA1 => checksums.sha1 = checksum_str, + ChecksumAlgorithm::SHA256 => checksums.sha256 = checksum_str, + ChecksumAlgorithm::CRC64NVME => checksums.crc64nvme = checksum_str, + _ => (), + } +} + +#[derive(Default)] +struct PutObjectChecksums { + crc32: Option, + crc32c: Option, + sha1: Option, + sha256: Option, + crc64nvme: Option, +} fn normalize_delete_objects_version_id(version_id: Option) -> Result<(Option, Option), String> { let version_id = version_id.map(|v| v.trim().to_string()).filter(|v| !v.is_empty()); @@ -129,67 +170,13 @@ fn normalize_delete_objects_version_id(version_id: Option) -> Result<(Op } } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PutObjectRequest { - pub bucket: String, - pub key: String, - pub content_length: Option, - pub content_type: Option, - pub version_id: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PutObjectResponse { - pub etag: String, - pub version_id: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct GetObjectRequest { - pub bucket: String, - pub key: String, - pub version_id: Option, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct GetObjectResponse { - pub etag: Option, - pub content_length: i64, - pub version_id: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DeleteObjectRequest { - pub bucket: String, - pub key: String, - pub version_id: Option, -} - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct DeleteObjectResponse { - pub delete_marker: Option, - pub version_id: Option, -} - -#[async_trait::async_trait] -pub trait ObjectUsecase: Send + Sync { - async fn put_object(&self, req: PutObjectRequest) -> ObjectUsecaseResult; - - async fn get_object(&self, req: GetObjectRequest) -> ObjectUsecaseResult; - - async fn delete_object(&self, req: DeleteObjectRequest) -> ObjectUsecaseResult; -} - #[derive(Clone, Default)] pub struct DefaultObjectUsecase { context: Option>, } impl DefaultObjectUsecase { - pub fn new(context: Arc) -> Self { - Self { context: Some(context) } - } - + #[cfg(test)] pub fn without_context() -> Self { Self { context: None } } @@ -200,10 +187,6 @@ impl DefaultObjectUsecase { } } - pub fn context(&self) -> Option> { - self.context.clone() - } - fn bucket_metadata_sys(&self) -> Option>> { self.context.as_ref().and_then(|context| context.bucket_metadata().handle()) } @@ -216,6 +199,35 @@ impl DefaultObjectUsecase { .unwrap_or_else(|| RustFSBufferConfig::default().base_config.default_unknown) } + async fn check_bucket_quota(&self, bucket: &str, op: QuotaOperation, size: u64) -> S3Result<()> { + let Some(metadata_sys) = self.bucket_metadata_sys() else { + return Ok(()); + }; + let quota_checker = QuotaChecker::new(metadata_sys); + match quota_checker.check_quota(bucket, op, size).await { + Ok(result) if !result.allowed => Err(S3Error::with_message( + S3ErrorCode::InvalidRequest, + format!( + "Bucket quota exceeded. Current usage: {} bytes, limit: {} bytes", + result.current_usage.unwrap_or(0), + result.quota_limit.unwrap_or(0) + ), + )), + Err(e) => { + warn!("Quota check failed for bucket {bucket}: {e}, allowing operation"); + Ok(()) + } + _ => Ok(()), + } + } + + fn spawn_cache_invalidation(bucket: String, key: String, version_id: Option) { + let manager = get_concurrency_manager(); + tokio::spawn(async move { + manager.invalidate_cache_versioned(&bucket, &key, version_id.as_deref()).await; + }); + } + #[instrument(level = "debug", skip(self, _fs, req))] pub async fn execute_put_object(&self, _fs: &FS, req: S3Request) -> S3Result> { if let Some(context) = &self.context { @@ -273,32 +285,9 @@ impl DefaultObjectUsecase { return Err(s3_error!(AccessDenied, "Access Denied")); } - // check quota for put operation - if let Some(size) = content_length - && let Some(metadata_sys) = self.bucket_metadata_sys() - { - let quota_checker = QuotaChecker::new(metadata_sys); - - match quota_checker - .check_quota(&bucket, QuotaOperation::PutObject, size as u64) - .await - { - Ok(check_result) => { - if !check_result.allowed { - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!( - "Bucket quota exceeded. Current usage: {} bytes, limit: {} bytes", - check_result.current_usage.unwrap_or(0), - check_result.quota_limit.unwrap_or(0) - ), - )); - } - } - Err(e) => { - warn!("Quota check failed for bucket {}: {}, allowing operation", bucket, e); - } - } + if let Some(size) = content_length { + self.check_bucket_quota(&bucket, QuotaOperation::PutObject, size as u64) + .await?; } let Some(body) = body else { return Err(s3_error!(IncompleteBody)) }; @@ -330,10 +319,6 @@ impl DefaultObjectUsecase { StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))), ); - // let body = Box::new(StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string()))))); - - // let mut reader = PutObjReader::new(body, content_length as usize); - let store = get_validated_store(&bucket).await?; // TDD: Get bucket default encryption configuration @@ -514,10 +499,6 @@ impl DefaultObjectUsecase { // Fast in-memory update for immediate quota consistency rustfs_ecstore::data_usage::increment_bucket_usage_memory(&bucket, obj_info.size as u64).await; - // Invalidate cache for the written object to prevent stale data - let manager = get_concurrency_manager(); - let put_bucket = bucket.clone(); - let put_key = key.clone(); let put_version = obj_info.version_id.map(|v| v.to_string()); helper = helper.object(obj_info.clone()); @@ -525,12 +506,7 @@ impl DefaultObjectUsecase { helper = helper.version_id(version_id.clone()); } - let put_version_clone = put_version.clone(); - tokio::spawn(async move { - manager - .invalidate_cache_versioned(&put_bucket, &put_key, put_version_clone.as_deref()) - .await; - }); + Self::spawn_cache_invalidation(bucket.clone(), key.clone(), put_version.clone()); let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)); @@ -543,77 +519,35 @@ impl DefaultObjectUsecase { schedule_replication(obj_info, store, dsc, ReplicationType::Object).await; } - let mut checksum_crc32 = input.checksum_crc32; - let mut checksum_crc32c = input.checksum_crc32c; - let mut checksum_sha1 = input.checksum_sha1; - let mut checksum_sha256 = input.checksum_sha256; - let mut checksum_crc64nvme = input.checksum_crc64nvme; - - if let Some(alg) = &input.checksum_algorithm - && let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| { - let key = match alg.as_str() { - ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(), - ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(), - ChecksumAlgorithm::SHA1 => rustfs_rio::ChecksumType::SHA1.key(), - ChecksumAlgorithm::SHA256 => rustfs_rio::ChecksumType::SHA256.key(), - ChecksumAlgorithm::CRC64NVME => rustfs_rio::ChecksumType::CRC64_NVME.key(), - _ => return None, - }; - trailer.read(|headers| { - headers - .get(key.unwrap_or_default()) - .and_then(|value| value.to_str().ok().map(|s| s.to_string())) - }) - }) - { - match alg.as_str() { - ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str, - ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str, - ChecksumAlgorithm::SHA1 => checksum_sha1 = checksum_str, - ChecksumAlgorithm::SHA256 => checksum_sha256 = checksum_str, - ChecksumAlgorithm::CRC64NVME => checksum_crc64nvme = checksum_str, - _ => (), - } - } + let mut checksums = PutObjectChecksums { + crc32: input.checksum_crc32, + crc32c: input.checksum_crc32c, + sha1: input.checksum_sha1, + sha256: input.checksum_sha256, + crc64nvme: input.checksum_crc64nvme, + }; + apply_trailing_checksums( + input.checksum_algorithm.as_ref().map(|a| a.as_str()), + &req.trailing_headers, + &mut checksums, + ); let output = PutObjectOutput { e_tag, - server_side_encryption: effective_sse, // TDD: Return effective encryption config + server_side_encryption: effective_sse, sse_customer_algorithm: sse_customer_algorithm.clone(), sse_customer_key_md5: sse_customer_key_md5.clone(), - ssekms_key_id: effective_kms_key_id, // TDD: Return effective KMS key ID - checksum_crc32, - checksum_crc32c, - checksum_sha1, - checksum_sha256, - checksum_crc64nvme, + ssekms_key_id: effective_kms_key_id, + checksum_crc32: checksums.crc32, + checksum_crc32c: checksums.crc32c, + checksum_sha1: checksums.sha1, + checksum_sha256: checksums.sha256, + checksum_crc64nvme: checksums.crc64nvme, version_id: put_version, ..Default::default() }; - // TODO fix response for POST Policy (multipart/form-data) ，wait s3s crate update,fix issue #1564 - // // If it is a POST Policy(multipart/form-data) path, the PutObjectInput carries the success_action_* field - // // Here, the response is uniformly rewritten, with the default being 204, redirect prioritizing 303, and status supporting 200/201/204 - // if input.success_action_status.is_some() || input.success_action_redirect.is_some() { - // let mut form_fields = HashMap::::new(); - // if let Some(v) = &input.success_action_status { - // form_fields.insert("success_action_status".to_string(), v.to_string()); - // } - // if let Some(v) = &input.success_action_redirect { - // form_fields.insert("success_action_redirect".to_string(), v.to_string()); - // } - // - // // obj_info.etag has been converted to e_tag (s3s etag) above, so try to pass the original string here - // let etag_str = e_tag.as_ref().map(|v| v.as_str()); - // - // // Returns using POST semantics: 204/303/201/200 - // let resp = build_post_object_success_response(&form_fields, &bucket, &key, etag_str, None)?; - // - // // Keep helper event complete (note: (StatusCode, Body) is returned here instead of PutObjectOutput) - // let result = Ok(resp); - // let _ = helper.complete(&result); - // return result; - // } + // TODO fix response for POST Policy (multipart/form-data), wait s3s crate update, fix issue #1564 let result = Ok(S3Response::new(output)); let _ = helper.complete(&result); @@ -1200,7 +1134,10 @@ impl DefaultObjectUsecase { // based on the wait time. Longer wait times indicate higher system // load, which triggers more conservative I/O parameters. let permit_wait_start = std::time::Instant::now(); - let _disk_permit = manager.acquire_disk_read_permit().await; + let _disk_permit = manager + .acquire_disk_read_permit() + .await + .map_err(|_| s3_error!(InternalError, "disk read semaphore closed"))?; let permit_wait_duration = permit_wait_start.elapsed(); // Calculate adaptive I/O strategy from permit wait time @@ -2107,34 +2044,9 @@ impl DefaultObjectUsecase { src_info.user_defined.insert(k, v); } - // check quota for copy operation - let has_bucket_metadata = if let Some(metadata_sys) = self.bucket_metadata_sys() { - let quota_checker = QuotaChecker::new(metadata_sys); - - match quota_checker - .check_quota(&bucket, QuotaOperation::CopyObject, src_info.size as u64) - .await - { - Ok(check_result) => { - if !check_result.allowed { - return Err(S3Error::with_message( - S3ErrorCode::InvalidRequest, - format!( - "Bucket quota exceeded. Current usage: {} bytes, limit: {} bytes", - check_result.current_usage.unwrap_or(0), - check_result.quota_limit.unwrap_or(0) - ), - )); - } - } - Err(e) => { - warn!("Quota check failed for bucket {}: {}, allowing operation", bucket, e); - } - } - true - } else { - false - }; + self.check_bucket_quota(&bucket, QuotaOperation::CopyObject, src_info.size as u64) + .await?; + let has_bucket_metadata = self.bucket_metadata_sys().is_some(); let oi = store .copy_object(&src_bucket, &src_key, &bucket, &key, &mut src_info, &src_opts, &dst_opts) @@ -2146,17 +2058,8 @@ impl DefaultObjectUsecase { rustfs_ecstore::data_usage::increment_bucket_usage_memory(&bucket, oi.size as u64).await; } - // Invalidate cache for the destination object to prevent stale data - let manager = get_concurrency_manager(); - let dest_bucket = bucket.clone(); - let dest_key = key.clone(); let dest_version = oi.version_id.map(|v| v.to_string()); - let dest_version_clone = dest_version.clone(); - tokio::spawn(async move { - manager - .invalidate_cache_versioned(&dest_bucket, &dest_key, dest_version_clone.as_deref()) - .await; - }); + Self::spawn_cache_invalidation(bucket.clone(), key.clone(), dest_version.clone()); // warn!("copy_object oi {:?}", &oi); let object_info = oi.clone(); @@ -2254,7 +2157,7 @@ impl DefaultObjectUsecase { }; { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = req_info_mut(&mut req)?; req_info.bucket = Some(bucket.clone()); req_info.object = Some(obj_id.key.clone()); req_info.version_id = version_id.clone(); @@ -2606,16 +2509,7 @@ impl DefaultObjectUsecase { // Fast in-memory update for immediate quota consistency rustfs_ecstore::data_usage::decrement_bucket_usage_memory(&bucket, obj_info.size as u64).await; - // Invalidate cache for the deleted object - let manager = get_concurrency_manager(); - let del_bucket = bucket.clone(); - let del_key = key.clone(); - let del_version = obj_info.version_id.map(|v| v.to_string()); - tokio::spawn(async move { - manager - .invalidate_cache_versioned(&del_bucket, &del_key, del_version.as_deref()) - .await; - }); + Self::spawn_cache_invalidation(bucket.clone(), key.clone(), obj_info.version_id.map(|v| v.to_string())); if obj_info.name.is_empty() { if replicate_force_delete { @@ -3489,49 +3383,30 @@ impl DefaultObjectUsecase { } } - let mut checksum_crc32 = input.checksum_crc32; - let mut checksum_crc32c = input.checksum_crc32c; - let mut checksum_sha1 = input.checksum_sha1; - let mut checksum_sha256 = input.checksum_sha256; - let mut checksum_crc64nvme = input.checksum_crc64nvme; - - if let Some(alg) = &input.checksum_algorithm - && let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| { - let key = match alg.as_str() { - ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(), - ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(), - ChecksumAlgorithm::SHA1 => rustfs_rio::ChecksumType::SHA1.key(), - ChecksumAlgorithm::SHA256 => rustfs_rio::ChecksumType::SHA256.key(), - ChecksumAlgorithm::CRC64NVME => rustfs_rio::ChecksumType::CRC64_NVME.key(), - _ => return None, - }; - trailer.read(|headers| { - headers - .get(key.unwrap_or_default()) - .and_then(|value| value.to_str().ok().map(|s| s.to_string())) - }) - }) - { - match alg.as_str() { - ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str, - ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str, - ChecksumAlgorithm::SHA1 => checksum_sha1 = checksum_str, - ChecksumAlgorithm::SHA256 => checksum_sha256 = checksum_str, - ChecksumAlgorithm::CRC64NVME => checksum_crc64nvme = checksum_str, - _ => (), - } - } + let mut checksums = PutObjectChecksums { + crc32: input.checksum_crc32, + crc32c: input.checksum_crc32c, + sha1: input.checksum_sha1, + sha256: input.checksum_sha256, + crc64nvme: input.checksum_crc64nvme, + }; + apply_trailing_checksums( + input.checksum_algorithm.as_ref().map(|a| a.as_str()), + &req.trailing_headers, + &mut checksums, + ); warn!( - "put object extract checksum_crc32={checksum_crc32:?}, checksum_crc32c={checksum_crc32c:?}, checksum_sha1={checksum_sha1:?}, checksum_sha256={checksum_sha256:?}, checksum_crc64nvme={checksum_crc64nvme:?}", + "put object extract checksum_crc32={:?}, checksum_crc32c={:?}, checksum_sha1={:?}, checksum_sha256={:?}, checksum_crc64nvme={:?}", + checksums.crc32, checksums.crc32c, checksums.sha1, checksums.sha256, checksums.crc64nvme, ); let output = PutObjectOutput { - checksum_crc32, - checksum_crc32c, - checksum_sha1, - checksum_sha256, - checksum_crc64nvme, + checksum_crc32: checksums.crc32, + checksum_crc32c: checksums.crc32c, + checksum_sha1: checksums.sha1, + checksum_sha256: checksums.sha256, + checksum_crc64nvme: checksums.crc64nvme, ..Default::default() }; let result = Ok(S3Response::new(output)); @@ -3540,30 +3415,6 @@ impl DefaultObjectUsecase { } } -#[async_trait::async_trait] -impl ObjectUsecase for DefaultObjectUsecase { - async fn put_object(&self, req: PutObjectRequest) -> ObjectUsecaseResult { - let _ = req; - Err(ApiError::from(StorageError::other( - "DefaultObjectUsecase::put_object DTO path is not implemented yet", - ))) - } - - async fn get_object(&self, req: GetObjectRequest) -> ObjectUsecaseResult { - let _ = req; - Err(ApiError::from(StorageError::other( - "DefaultObjectUsecase::get_object DTO path is not implemented yet", - ))) - } - - async fn delete_object(&self, req: DeleteObjectRequest) -> ObjectUsecaseResult { - let _ = req; - Err(ApiError::from(StorageError::other( - "DefaultObjectUsecase::delete_object DTO path is not implemented yet", - ))) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/rustfs/src/config/workload_profiles.rs b/rustfs/src/config/workload_profiles.rs index 79b8a46ef..a28afd840 100644 --- a/rustfs/src/config/workload_profiles.rs +++ b/rustfs/src/config/workload_profiles.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - //! Adaptive buffer sizing optimization for different workload types. //! //! This module provides intelligent buffer size selection based on file size and workload profile @@ -86,6 +84,7 @@ pub enum WorkloadProfile { /// Secure storage: security first, memory constrained for compliance SecureStorage, /// Custom configuration for specialized requirements + #[allow(dead_code)] Custom(BufferConfig), } @@ -107,6 +106,7 @@ pub struct BufferConfig { #[derive(Debug, Clone)] pub struct RustFSBufferConfig { /// Selected workload profile + #[allow(dead_code)] pub workload: WorkloadProfile, /// Computed buffer configuration (either from profile or custom) pub base_config: BufferConfig, @@ -309,6 +309,7 @@ impl BufferConfig { } /// Validate the buffer configuration + #[allow(dead_code)] pub fn validate(&self) -> Result<(), String> { if self.min_size == 0 { return Err("min_size must be greater than 0".to_string()); diff --git a/rustfs/src/license.rs b/rustfs/src/license.rs index ce62e3b32..01b42fdb8 100644 --- a/rustfs/src/license.rs +++ b/rustfs/src/license.rs @@ -55,21 +55,6 @@ pub fn license_check() -> Result<()> { Ok(()) }); - // let invalid_license = config::get_config().license.as_ref().map(|license| { - // if license.is_empty() { - // error!("License is empty"); - // return Err(Error::other("Incorrect license, please contact RustFS.".to_string())); - // } - // let token = appauth::token::parse_license(license)?; - // if token.expired < SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs() { - // error!("License expired"); - // return Err(Error::other("Incorrect license, please contact RustFS.".to_string())); - // } - - // info!("License is valid ! expired at {}", token.expired); - // Ok(()) - // }); - if invalid_license.is_none() || invalid_license.is_some_and(|v| v.is_err()) { return Err(Error::other("Incorrect license, please contact RustFS.")); } diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 82c6d58b8..67d28c5b6 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -46,7 +46,6 @@ use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr}; use rustfs_credentials::init_global_action_credentials; use rustfs_ecstore::store::init_lock_clients; use rustfs_ecstore::{ - StorageAPI, bucket::metadata_sys::init_bucket_metadata_sys, bucket::replication::{get_global_replication_pool, init_background_replication}, config as ecconfig, @@ -56,6 +55,7 @@ use rustfs_ecstore::{ set_global_endpoints, store::ECStore, store::init_local_disks, + store_api::BucketOperations, store_api::BucketOptions, update_erasure_type, }; @@ -94,8 +94,8 @@ fn main() { .expect("Failed to build Tokio runtime"); let result = runtime.block_on(async_main()); if let Err(ref e) = result { - eprintln!("{} Server encountered an error and is shutting down: {}", jiff::Zoned::now(), e); - error!("Server encountered an error and is shutting down: {}", e); + // Use eprintln as tracing may not be initialized at this point + eprintln!("[FATAL] Server encountered an error and is shutting down: {e}"); std::process::exit(1); } } @@ -110,7 +110,8 @@ async fn async_main() -> Result<()> { let guard = match init_obs(Some(config.clone().obs_endpoint)).await { Ok(g) => g, Err(e) => { - println!("Failed to initialize observability: {e}"); + // Use eprintln as tracing is not yet initialized + eprintln!("[FATAL] Failed to initialize observability: {e}"); return Err(Error::other(e)); } }; @@ -434,13 +435,13 @@ async fn run(config: config::Config) -> Result<()> { init_metrics_system(ctx.clone()); } - println!( + info!( + target: "rustfs::main::run", "RustFS server version: {} started successfully at {}, current time: {}", version::get_version(), &server_address, jiff::Zoned::now() ); - info!(target: "rustfs::main::run","server started successfully at {}", &server_address); // 4. Mark as Full Ready now that critical components are warm readiness.mark_stage(SystemStage::FullReady); @@ -580,9 +581,5 @@ async fn handle_shutdown( // the last updated status is stopped state_manager.update(ServiceState::Stopped); - info!( - target: "rustfs::main::handle_shutdown", - "Server stopped current " - ); - println!("Server stopped successfully."); + info!(target: "rustfs::main::handle_shutdown", "Server stopped successfully."); } diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index a914cdfe6..6bb98aac2 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -187,14 +187,9 @@ pub async fn start_http_server( "Console WebUI (localhost): {protocol}://127.0.0.1:{server_port}/rustfs/console/index.html", ); - - println!("Console WebUI Start Time: {now_time}"); - println!("Console WebUI available at: {protocol}://{local_ip_str}:{server_port}/rustfs/console/index.html"); - println!("Console WebUI (localhost): {protocol}://127.0.0.1:{server_port}/rustfs/console/index.html"); } else { - info!(target: "rustfs::main::startup","RustFS API: {api_endpoints} {localhost_endpoint}"); - println!("RustFS Http API: {api_endpoints} {localhost_endpoint}"); - println!("RustFS Start Time: {now_time}"); + info!(target: "rustfs::main::startup", "RustFS API: {api_endpoints} {localhost_endpoint}"); + info!(target: "rustfs::main::startup", "RustFS Start Time: {now_time}"); if rustfs_credentials::DEFAULT_ACCESS_KEY.eq(&config.access_key) && rustfs_credentials::DEFAULT_SECRET_KEY.eq(&config.secret_key) { diff --git a/rustfs/src/server/runtime.rs b/rustfs/src/server/runtime.rs index 6a288a55f..a82fca49c 100644 --- a/rustfs/src/server/runtime.rs +++ b/rustfs/src/server/runtime.rs @@ -129,29 +129,23 @@ pub(crate) fn get_tokio_runtime_builder() -> tokio::runtime::Builder { if print_tokio_thread_enable() { builder .on_thread_start(|| { - let id = std::thread::current().id(); - println!( - "RustFS Worker Thread running - initializing resources time: {:?}, thread id: {:?}", - jiff::Zoned::now().to_string(), - id - ); + tracing::trace!(thread_id = ?std::thread::current().id(), "worker thread started"); }) .on_thread_stop(|| { - let id = std::thread::current().id(); - println!( - "RustFS Worker Thread stopping - cleaning up resources time: {:?}, thread id: {:?}", - jiff::Zoned::now().to_string(), - id - ) + tracing::trace!(thread_id = ?std::thread::current().id(), "worker thread stopped"); }); } if !rustfs_obs::is_production_environment() { - println!( - "Starting Tokio runtime with configured parameters:\n\ - worker_threads: {worker_threads}, max_blocking_threads: {max_blocking_threads}, \ - thread_stack_size: {thread_stack_size}, thread_keep_alive: {thread_keep_alive}, \ - global_queue_interval: {global_queue_interval}, event_interval: {event_interval}, \ - max_io_events_per_tick: {max_io_events_per_tick}, thread_name: {thread_name}" + tracing::debug!( + worker_threads, + max_blocking_threads, + thread_stack_size, + thread_keep_alive, + global_queue_interval, + event_interval, + max_io_events_per_tick, + thread_name, + "Starting Tokio runtime with configured parameters" ); } builder diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index 9f7c68915..88acd5068 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -24,11 +24,11 @@ use crate::error::ApiError; use crate::license::license_check; use crate::server::RemoteAddr; use metrics::counter; -use rustfs_ecstore::StorageAPI; use rustfs_ecstore::bucket::metadata_sys; use rustfs_ecstore::bucket::policy_sys::PolicySys; use rustfs_ecstore::error::StorageError; use rustfs_ecstore::new_object_layer_fn; +use rustfs_ecstore::store_api::ObjectOperations; use rustfs_iam::error::Error as IamError; use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_policy::policy::{Args, BucketPolicyArgs}; @@ -37,7 +37,6 @@ use s3s::access::{S3Access, S3AccessContext}; use s3s::{S3Error, S3ErrorCode, S3Request, S3Result, dto::*, s3_error}; use std::collections::HashMap; -#[allow(dead_code)] #[derive(Default, Clone, Debug)] pub(crate) struct ReqInfo { pub cred: Option, @@ -45,6 +44,7 @@ pub(crate) struct ReqInfo { pub bucket: Option, pub object: Option, pub version_id: Option, + #[allow(dead_code)] pub region: Option, } @@ -201,6 +201,23 @@ async fn check_acl_access(req: &S3Request, req_info: &ReqInfo, action: &Ac Ok(acl_allows(&acl, user_id, is_authenticated, permission, ignore_public_acls)) } +pub(crate) fn req_info_ref(req: &S3Request) -> S3Result<&ReqInfo> { + req.extensions + .get::() + .ok_or_else(|| s3_error!(InternalError, "ReqInfo not found in request extensions")) +} + +pub(crate) fn req_info_mut(req: &mut S3Request) -> S3Result<&mut ReqInfo> { + req.extensions + .get_mut::() + .ok_or_else(|| s3_error!(InternalError, "ReqInfo not found in request extensions")) +} + +fn ext_req_info_mut(ext: &mut http::Extensions) -> S3Result<&mut ReqInfo> { + ext.get_mut::() + .ok_or_else(|| s3_error!(InternalError, "ReqInfo not found in request extensions")) +} + #[derive(Clone, Debug)] pub(crate) struct ObjectTagConditions(pub HashMap>); @@ -241,7 +258,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R let remote_addr = req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)); let object_tag_conditions = req.extensions.get::().cloned(); - let req_info = req.extensions.get::().expect("ReqInfo not found"); + let req_info = req_info_ref(req)?; if let Some(cred) = &req_info.cred { let Ok(iam_store) = rustfs_iam::get() else { @@ -550,7 +567,7 @@ impl S3Access for FS { async fn create_bucket(&self, req: &mut S3Request) -> S3Result<()> { license_check().map_err(|er| s3_error!(AccessDenied, "{:?}", er.to_string()))?; - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::CreateBucketAction)).await?; @@ -588,7 +605,7 @@ impl S3Access for FS { } }; - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(src_bucket.clone()); req_info.object = Some(src_key.clone()); req_info.version_id = version_id.clone(); @@ -601,7 +618,7 @@ impl S3Access for FS { authorize_request(req, Action::S3Action(S3Action::GetObjectAction)).await?; } - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); @@ -622,7 +639,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn delete_bucket(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::DeleteBucketAction)).await?; @@ -647,7 +664,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn delete_bucket_cors(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::DeleteBucketCorsAction)).await @@ -657,7 +674,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn delete_bucket_encryption(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutBucketEncryptionAction)).await @@ -687,7 +704,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn delete_bucket_lifecycle(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutBucketLifecycleAction)).await @@ -714,7 +731,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn delete_bucket_policy(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::DeleteBucketPolicyAction)).await @@ -724,7 +741,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn delete_bucket_replication(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutReplicationConfigurationAction)).await @@ -734,7 +751,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn delete_bucket_tagging(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutBucketTaggingAction)).await @@ -751,7 +768,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn delete_object(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -775,7 +792,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn delete_object_tagging(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -797,7 +814,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn delete_objects(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = None; req_info.version_id = None; @@ -816,7 +833,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn delete_public_access_block(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::DeleteBucketPublicAccessBlockAction)).await @@ -836,7 +853,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_bucket_acl(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::GetBucketAclAction)).await @@ -856,7 +873,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_bucket_cors(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::GetBucketCorsAction)).await @@ -866,7 +883,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_bucket_encryption(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::GetBucketEncryptionAction)).await @@ -899,7 +916,7 @@ impl S3Access for FS { &self, req: &mut S3Request, ) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::GetBucketLifecycleAction)).await @@ -909,7 +926,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_bucket_location(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::GetBucketLocationAction)).await @@ -936,7 +953,7 @@ impl S3Access for FS { &self, req: &mut S3Request, ) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::GetBucketNotificationAction)).await @@ -953,7 +970,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_bucket_policy(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, get_bucket_policy_authorize_action()).await @@ -963,7 +980,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_bucket_policy_status(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::GetBucketPolicyStatusAction)).await @@ -973,7 +990,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_bucket_replication(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::GetReplicationConfigurationAction)).await @@ -990,7 +1007,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_bucket_tagging(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::GetBucketTaggingAction)).await @@ -1000,7 +1017,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_bucket_versioning(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::GetBucketVersioningAction)).await @@ -1017,7 +1034,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_object(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -1034,7 +1051,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_object_acl(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -1051,7 +1068,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_object_attributes(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -1081,7 +1098,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_object_legal_hold(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -1093,7 +1110,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_object_lock_configuration(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::GetBucketObjectLockConfigurationAction)).await @@ -1103,7 +1120,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_object_retention(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -1115,7 +1132,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_object_tagging(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -1139,7 +1156,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn get_public_access_block(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::GetBucketPublicAccessBlockAction)).await @@ -1149,7 +1166,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn head_bucket(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::ListBucketAction)).await @@ -1159,7 +1176,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn head_object(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -1224,7 +1241,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn list_multipart_uploads(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::ListBucketMultipartUploadsAction)).await @@ -1235,7 +1252,7 @@ impl S3Access for FS { /// Returns `Ok(())` if the request is allowed, or an error if access is denied or another /// authorization-related issue occurs. async fn list_object_versions(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::ListBucketVersionsAction)).await } @@ -1244,7 +1261,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn list_objects(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::ListBucketAction)).await @@ -1254,7 +1271,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn list_objects_v2(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::ListBucketAction)).await @@ -1281,7 +1298,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn put_bucket_acl(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutBucketAclAction)).await @@ -1301,7 +1318,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn put_bucket_cors(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutBucketCorsAction)).await @@ -1311,7 +1328,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn put_bucket_encryption(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutBucketEncryptionAction)).await @@ -1344,7 +1361,7 @@ impl S3Access for FS { &self, req: &mut S3Request, ) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutBucketLifecycleAction)).await @@ -1371,7 +1388,7 @@ impl S3Access for FS { &self, req: &mut S3Request, ) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutBucketNotificationAction)).await @@ -1388,7 +1405,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn put_bucket_policy(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, put_bucket_policy_authorize_action()).await @@ -1398,7 +1415,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn put_bucket_replication(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutReplicationConfigurationAction)).await @@ -1415,7 +1432,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn put_bucket_tagging(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutBucketTaggingAction)).await @@ -1425,7 +1442,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn put_bucket_versioning(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutBucketVersioningAction)).await @@ -1444,7 +1461,7 @@ impl S3Access for FS { async fn put_object(&self, req: &mut S3Request) -> S3Result<()> { license_check().map_err(|er| s3_error!(AccessDenied, "{:?}", er.to_string()))?; - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -1456,7 +1473,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn put_object_acl(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -1473,7 +1490,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn put_object_legal_hold(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -1485,7 +1502,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn put_object_lock_configuration(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutBucketObjectLockConfigurationAction)).await @@ -1493,7 +1510,7 @@ impl S3Access for FS { /// Checks whether the PutObjectRetention request has accesses to the resources. async fn put_object_retention(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -1512,7 +1529,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn put_object_tagging(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -1529,7 +1546,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn put_public_access_block(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); authorize_request(req, Action::S3Action(S3Action::PutBucketPublicAccessBlockAction)).await @@ -1539,7 +1556,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn restore_object(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); req_info.version_id = req.input.version_id.clone(); @@ -1551,7 +1568,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn select_object_content(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); @@ -1562,7 +1579,7 @@ impl S3Access for FS { /// /// This method returns `Ok(())` by default. async fn upload_part(&self, req: &mut S3Request) -> S3Result<()> { - let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); + let req_info = ext_req_info_mut(&mut req.extensions)?; req_info.bucket = Some(req.input.bucket.clone()); req_info.object = Some(req.input.key.clone()); diff --git a/rustfs/src/storage/concurrency.rs b/rustfs/src/storage/concurrency.rs index e29ad1fd0..706a593f9 100644 --- a/rustfs/src/storage/concurrency.rs +++ b/rustfs/src/storage/concurrency.rs @@ -134,7 +134,6 @@ impl IoLoadLevel { /// let enable_readahead = strategy.enable_readahead; /// let enable_cache_writeback = strategy.cache_writeback_enabled; /// ``` -#[allow(dead_code)] #[derive(Debug, Clone)] pub struct IoStrategy { /// Recommended buffer size for I/O operations (in bytes). @@ -164,6 +163,7 @@ pub struct IoStrategy { /// Whether to use tokio BufReader for improved async I/O. /// /// Always enabled for better async performance. + #[allow(dead_code)] pub use_buffered_io: bool, /// The detected I/O load level. @@ -668,7 +668,6 @@ struct CachedObject { /// }; /// manager.put_cached_object(cache_key, cached).await; /// ``` -#[allow(dead_code)] #[derive(Clone, Debug)] pub struct CachedGetObject { /// The object body data @@ -682,6 +681,7 @@ pub struct CachedGetObject { /// Last modified time as RFC3339 string (e.g., "2024-01-01T12:00:00Z") pub last_modified: Option, /// Expiration time as RFC3339 string + #[allow(dead_code)] pub expires: Option, /// Cache-Control header value pub cache_control: Option, @@ -700,10 +700,12 @@ pub struct CachedGetObject { /// Number of tags associated with the object pub tag_count: Option, /// Replication status + #[allow(dead_code)] pub replication_status: Option, /// User-defined metadata (x-amz-meta-*) pub user_metadata: std::collections::HashMap, /// When this object was cached (for internal use, automatically set) + #[allow(dead_code)] cached_at: Option, /// Access count for hot key tracking (automatically managed) access_count: Arc, @@ -1320,11 +1322,8 @@ impl ConcurrencyManager { /// /// This ensures we don't overwhelm the disk subsystem with too many /// concurrent reads, which can cause performance degradation. - pub async fn acquire_disk_read_permit(&self) -> tokio::sync::SemaphorePermit<'_> { - self.disk_read_semaphore - .acquire() - .await - .expect("semaphore closed unexpectedly") + pub async fn acquire_disk_read_permit(&self) -> Result, tokio::sync::AcquireError> { + self.disk_read_semaphore.acquire().await } // ============================================ @@ -1791,10 +1790,10 @@ mod tests { async fn test_disk_read_permits() { let manager = ConcurrencyManager::new(); - let permit1 = manager.acquire_disk_read_permit().await; + let permit1 = manager.acquire_disk_read_permit().await.unwrap(); assert_eq!(manager.disk_read_semaphore.available_permits(), 63); - let permit2 = manager.acquire_disk_read_permit().await; + let permit2 = manager.acquire_disk_read_permit().await.unwrap(); assert_eq!(manager.disk_read_semaphore.available_permits(), 62); drop(permit1); diff --git a/rustfs/src/storage/concurrent_get_object_test.rs b/rustfs/src/storage/concurrent_get_object_test.rs index f6e83f90f..c0d6ef084 100644 --- a/rustfs/src/storage/concurrent_get_object_test.rs +++ b/rustfs/src/storage/concurrent_get_object_test.rs @@ -258,7 +258,7 @@ mod tests { .map(|_| { let mgr = Arc::new(manager.clone()); tokio::spawn(async move { - let _permit = mgr.acquire_disk_read_permit().await; + let _permit = mgr.acquire_disk_read_permit().await.unwrap(); sleep(Duration::from_millis(10)).await; }) }) diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 2a7680cee..0db5edee1 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -16,11 +16,10 @@ use crate::app::bucket_usecase::DefaultBucketUsecase; use crate::app::multipart_usecase::DefaultMultipartUsecase; use crate::app::object_usecase::DefaultObjectUsecase; use rustfs_ecstore::{ - StorageAPI, bucket::tagging::decode_tags_to_map, error::{is_err_object_not_found, is_err_version_not_found}, new_object_layer_fn, - store_api::ObjectOptions, + store_api::{ObjectOperations, ObjectOptions}, }; use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error}; use serde::{Deserialize, Serialize}; diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index eab2374bd..c01e41d71 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -25,8 +25,8 @@ use rustfs_ecstore::bucket::metadata_sys::get_replication_config; use rustfs_ecstore::bucket::object_lock::objectlock_sys; use rustfs_ecstore::bucket::replication::ReplicationConfigurationExt; use rustfs_ecstore::error::StorageError; -use rustfs_ecstore::store_api::{BucketOptions, ObjectInfo, ObjectToDelete}; -use rustfs_ecstore::{StorageAPI, new_object_layer_fn}; +use rustfs_ecstore::new_object_layer_fn; +use rustfs_ecstore::store_api::{BucketOperations, BucketOptions, ObjectInfo, ObjectToDelete}; use rustfs_targets::EventName; use rustfs_targets::arn::{TargetID, TargetIDError}; use rustfs_utils::http::{ diff --git a/rustfs/src/storage/entity.rs b/rustfs/src/storage/entity.rs index 7273100db..5d8f9cb7f 100644 --- a/rustfs/src/storage/entity.rs +++ b/rustfs/src/storage/entity.rs @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(dead_code)] - use s3s::dto::{ BucketKeyEnabled, BucketName, ChecksumCRC32, ChecksumCRC32C, ChecksumCRC64NVME, ChecksumSHA1, ChecksumSHA256, ChecksumType, ETag, Expiration, Location, ObjectKey, ObjectVersionId, RequestCharged, SSEKMSKeyId, ServerSideEncryption, }; +#[allow(dead_code)] #[derive(Debug, Clone, Default)] pub struct CompleteMultipartUploadOutput { pub bucket: Option, diff --git a/rustfs/src/storage/head_prefix.rs b/rustfs/src/storage/head_prefix.rs index efa0badbc..651c0dc36 100644 --- a/rustfs/src/storage/head_prefix.rs +++ b/rustfs/src/storage/head_prefix.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use rustfs_ecstore::StorageAPI; use rustfs_ecstore::store::ECStore; +use rustfs_ecstore::store_api::ListOperations; use std::sync::Arc; /// Determines if the key "looks like a prefix" (ends with `/`). diff --git a/rustfs/src/storage/rpc/disk.rs b/rustfs/src/storage/rpc/disk.rs index a7f804d93..d1f469f2d 100644 --- a/rustfs/src/storage/rpc/disk.rs +++ b/rustfs/src/storage/rpc/disk.rs @@ -633,39 +633,6 @@ impl NodeService { pub(super) async fn handle_write(&self, _request: Request) -> Result, Status> { unimplemented!("write"); - // let request = request.into_inner(); - // if let Some(disk) = self.find_disk(&request.disk).await { - // let file_writer = if request.is_append { - // disk.append_file(&request.volume, &request.path).await - // } else { - // disk.create_file("", &request.volume, &request.path, 0).await - // }; - - // match file_writer { - // Ok(mut file_writer) => match file_writer.write(&request.data).await { - // Ok(_) => Ok(Response::new(WriteResponse { - // success: true, - // error: None, - // })), - // Err(err) => Ok(Response::new(WriteResponse { - // success: false, - // error: Some(err_to_proto_err(&err, &format!("write failed: {}", err))), - // })), - // }, - // Err(err) => Ok(Response::new(WriteResponse { - // success: false, - // error: Some(err_to_proto_err(&err, &format!("get writer failed: {}", err))), - // })), - // } - // } else { - // Ok(Response::new(WriteResponse { - // success: false, - // error: Some(err_to_proto_err( - // &EcsError::new(StorageError::InvalidArgument(Default::default(), Default::default(), Default::default())), - // "can not find disk", - // )), - // })) - // } } pub(super) async fn handle_rename_file( diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index eb12cb9bb..ec2088922 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -63,26 +63,6 @@ mod lock; #[path = "metrics.rs"] mod metrics; -// fn match_for_io_error(err_status: &Status) -> Option<&std::io::Error> { -// let mut err: &(dyn Error + 'static) = err_status; - -// loop { -// if let Some(io_err) = err.downcast_ref::() { -// return Some(io_err); -// } - -// // h2::Error do not expose std::io::Error with `source()` -// // https://github.com/hyperium/h2/pull/462 -// if let Some(h2_err) = err.downcast_ref::() { -// if let Some(io_err) = h2_err.get_io() { -// return Some(io_err); -// } -// } - -// err = err.source()?; -// } -// } - #[derive(Debug)] pub struct NodeService { local_peer: LocalPeerS3Client, @@ -198,200 +178,12 @@ impl Node for NodeService { let _ = request; unimplemented!("write_stream"); - - // let mut in_stream = request.into_inner(); - // let (tx, rx) = mpsc::channel(128); - - // tokio::spawn(async move { - // let mut file_ref = None; - // while let Some(result) = in_stream.next().await { - // match result { - // // Ok(v) => tx - // // .send(Ok(EchoResponse { message: v.message })) - // // .await - // // .expect("working rx"), - // Ok(v) => { - // match file_ref.as_ref() { - // Some(_) => (), - // None => { - // if let Some(disk) = find_local_disk(&v.disk).await { - // let file_writer = if v.is_append { - // disk.append_file(&v.volume, &v.path).await - // } else { - // disk.create_file("", &v.volume, &v.path, 0).await - // }; - - // match file_writer { - // Ok(file_writer) => file_ref = Some(file_writer), - // Err(err) => { - // tx.send(Ok(WriteResponse { - // success: false, - // error: Some(err_to_proto_err( - // &err, - // &format!("get file writer failed: {}", err), - // )), - // })) - // .await - // .expect("working rx"); - // break; - // } - // } - // } else { - // tx.send(Ok(WriteResponse { - // success: false, - // error: Some(err_to_proto_err( - // &EcsError::new(StorageError::InvalidArgument( - // Default::default(), - // Default::default(), - // Default::default(), - // )), - // "can not find disk", - // )), - // })) - // .await - // .expect("working rx"); - // break; - // } - // } - // }; - - // match file_ref.as_mut().unwrap().write(&v.data).await { - // Ok(_) => tx.send(Ok(WriteResponse { - // success: true, - // error: None, - // })), - // Err(err) => tx.send(Ok(WriteResponse { - // success: false, - // error: Some(err_to_proto_err(&err, &format!("write failed: {}", err))), - // })), - // } - // .await - // .unwrap(); - // } - // Err(err) => { - // if let Some(io_err) = match_for_io_error(&err) { - // if io_err.kind() == ErrorKind::BrokenPipe { - // // here you can handle special case when client - // // disconnected in unexpected way - // eprintln!("\tclient disconnected: broken pipe"); - // break; - // } - // } - - // match tx.send(Err(err)).await { - // Ok(_) => (), - // Err(_err) => break, // response was dropped - // } - // } - // } - // } - // println!("\tstream ended"); - // }); - - // let out_stream = ReceiverStream::new(rx); - - // Ok(Response::new(Box::pin(out_stream))) } type ReadAtStream = ResponseStream; async fn read_at(&self, _request: Request>) -> Result, Status> { info!("read_at"); unimplemented!("read_at"); - - // let mut in_stream = request.into_inner(); - // let (tx, rx) = mpsc::channel(128); - - // tokio::spawn(async move { - // let mut file_ref = None; - // while let Some(result) = in_stream.next().await { - // match result { - // Ok(v) => { - // match file_ref.as_ref() { - // Some(_) => (), - // None => { - // if let Some(disk) = find_local_disk(&v.disk).await { - // match disk.read_file(&v.volume, &v.path).await { - // Ok(file_reader) => file_ref = Some(file_reader), - // Err(err) => { - // tx.send(Ok(ReadAtResponse { - // success: false, - // data: Vec::new(), - // error: Some(err_to_proto_err(&err, &format!("read file failed: {}", err))), - // read_size: -1, - // })) - // .await - // .expect("working rx"); - // break; - // } - // } - // } else { - // tx.send(Ok(ReadAtResponse { - // success: false, - // data: Vec::new(), - // error: Some(err_to_proto_err( - // &EcsError::new(StorageError::InvalidArgument( - // Default::default(), - // Default::default(), - // Default::default(), - // )), - // "can not find disk", - // )), - // read_size: -1, - // })) - // .await - // .expect("working rx"); - // break; - // } - // } - // }; - - // let mut data = vec![0u8; v.length.try_into().unwrap()]; - - // match file_ref - // .as_mut() - // .unwrap() - // .read_at(v.offset.try_into().unwrap(), &mut data) - // .await - // { - // Ok(read_size) => tx.send(Ok(ReadAtResponse { - // success: true, - // data, - // read_size: read_size.try_into().unwrap(), - // error: None, - // })), - // Err(err) => tx.send(Ok(ReadAtResponse { - // success: false, - // data: Vec::new(), - // error: Some(err_to_proto_err(&err, &format!("read at failed: {}", err))), - // read_size: -1, - // })), - // } - // .await - // .unwrap(); - // } - // Err(err) => { - // if let Some(io_err) = match_for_io_error(&err) { - // if io_err.kind() == ErrorKind::BrokenPipe { - // // here you can handle special case when client - // // disconnected in unexpected way - // eprintln!("\tclient disconnected: broken pipe"); - // break; - // } - // } - - // match tx.send(Err(err)).await { - // Ok(_) => (), - // Err(_err) => break, // response was dropped - // } - // } - // } - // } - // println!("\tstream ended"); - // }); - - // let out_stream = ReceiverStream::new(rx); - - // Ok(Response::new(Box::pin(out_stream))) } async fn list_dir(&self, request: Request) -> Result, Status> { diff --git a/rustfs/src/storage/s3_api/mod.rs b/rustfs/src/storage/s3_api/mod.rs index 3de017587..156c28e44 100644 --- a/rustfs/src/storage/s3_api/mod.rs +++ b/rustfs/src/storage/s3_api/mod.rs @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![allow(dead_code)] + //! Facade modules for incremental S3 API extraction from `ecfs.rs`. //! //! This file intentionally starts as skeleton-only. Behavior remains in place //! until each helper is moved with dedicated small refactor steps. -#![allow(dead_code)] pub(crate) mod acl; pub(crate) mod bucket; diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index a1ff8b6dc..2a45d3742 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -684,7 +684,7 @@ async fn apply_ssec_prepare_encryption_material( Ok(EncryptionMaterial { sse_type: SSEType::SseC, - server_side_encryption: ServerSideEncryption::AES256.parse().unwrap(), + server_side_encryption: ServerSideEncryption::from_static(ServerSideEncryption::AES256), kms_key_id: None, algorithm, key_bytes: [0; 32], @@ -731,7 +731,7 @@ async fn apply_ssec_encryption_material( Ok(EncryptionMaterial { sse_type: SSEType::SseC, - server_side_encryption: ServerSideEncryption::AES256.parse().unwrap(), + server_side_encryption: ServerSideEncryption::from_static(ServerSideEncryption::AES256), kms_key_id: None, algorithm: validated.algorithm, key_bytes: validated.key_bytes, @@ -776,7 +776,7 @@ async fn apply_ssec_decryption_material( Ok(DecryptionMaterial { sse_type: SSEType::SseC, - server_side_encryption: ServerSideEncryption::AES256.parse().unwrap(), // const + server_side_encryption: ServerSideEncryption::from_static(ServerSideEncryption::AES256), // const kms_key_id: None, algorithm: SSECustomerAlgorithm::from(algorithm), @@ -941,7 +941,8 @@ async fn apply_managed_decryption_material( return Ok(None); } - let server_side_encryption = metadata.get("x-amz-server-side-encryption").unwrap().clone(); + // Safe: presence is guaranteed by the contains_key check above. + let server_side_encryption = metadata.get("x-amz-server-side-encryption").cloned().unwrap_or_default(); // Parse metadata - try using service if available, otherwise parse manually let (encrypted_data_key, iv, algorithm) = if let Some(service) = get_global_encryption_service().await { @@ -1169,17 +1170,17 @@ impl TestSseDekProvider { let decoded_len = v.len(); match v.try_into() { Ok(arr) => { - println!("✓ Successfully loaded master key (32 bytes)"); + tracing::info!("Successfully loaded SSE master key (32 bytes)"); arr } Err(_) => { - eprintln!("✗ Failed to load master key: decoded key is not 32 bytes (got {} bytes)", decoded_len); + tracing::error!("Failed to load master key: decoded key is not 32 bytes (got {decoded_len} bytes)"); [0u8; 32] } } } Err(e) => { - eprintln!("✗ Failed to load master key: invalid base64 encoding: {}", e); + tracing::error!("Failed to load master key: invalid base64 encoding: {e}"); [0u8; 32] } } @@ -1188,8 +1189,9 @@ impl TestSseDekProvider { }; if master_key == [0u8; 32] { - eprintln!("✗ Failed to load master key: no valid master key loaded! All encryption operations will fail."); - eprintln!(" Set __RUSTFS_SSE_SIMPLE_CMK environment variable to a base64-encoded 32-byte key."); + tracing::error!( + "No valid SSE master key loaded. Set __RUSTFS_SSE_SIMPLE_CMK environment variable to a base64-encoded 32-byte key." + ); std::process::exit(1); } @@ -1459,8 +1461,9 @@ pub fn validate_ssec_params(params: SsecParams) -> Resultinfra -cycle|infra<->interface -dep|rustfs/src/main.rs|infra->app|crate::app::context:: -dep|rustfs/src/server/audit.rs|infra->app|crate::app::context::resolve_server_config -dep|rustfs/src/server/event.rs|infra->app|crate::app::context::resolve_server_config -dep|rustfs/src/server/http.rs|infra->interface|crate::admin -dep|rustfs/src/server/layer.rs|infra->interface|crate::admin::console::is_console_path -dep|rustfs/src/storage/objects/put_object.rs|infra->app|crate::app::context::resolve_bucket_metadata_handle +# Layer dependency baseline for the rustfs binary crate. +# +# These are intra-crate module references within rustfs/ that cross the +# conceptual layer boundaries (app, infra/server, interface/admin). +# Since they live inside one Cargo crate, Rust doesn't enforce separation. +# The list is maintained for architectural awareness during code review. +# +# Format: status|source_file|direction|imported_symbol +# +# Status: +# accepted - reviewed and intentionally allowed +# todo - should be resolved in a future refactor + +accepted|rustfs/src/main.rs|infra->app|crate::app::context::*|main wires all layers +accepted|rustfs/src/server/audit.rs|infra->app|crate::app::context::resolve_server_config|config resolution uses global AppContext +accepted|rustfs/src/server/event.rs|infra->app|crate::app::context::resolve_server_config|config resolution uses global AppContext +accepted|rustfs/src/server/http.rs|infra->interface|crate::admin|HTTP server routes to admin handlers +accepted|rustfs/src/server/layer.rs|infra->interface|crate::admin::console::is_console_path|pure path predicate for routing +accepted|rustfs/src/storage/objects/put_object.rs|infra->app|crate::app::context::resolve_bucket_metadata_handle|metadata resolution uses global AppContext