gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-11 07:36:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: route ecstore storage api boundaries (#3892)

Browse Source
This commit is contained in:
Zhengchao An
2026-06-26 09:35:18 +08:00
committed by GitHub
parent 0b9c8a7731
commit c9614eb7cb
56 changed files with 389 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/benches/comparison_benchmark.rs
+2 -1
View File
@@ -33,9 +33,10 @@
//! ```
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
use rustfs_ecstore::api::erasure::Erasure;
mod storage_api;
use std::hint::black_box;
use std::time::Duration;
use storage_api::Erasure;
/// Performance test data configuration
struct TestData {
crates/ecstore/benches/erasure_benchmark.rs
+2 -1
View File
@@ -44,11 +44,12 @@
//! - SIMD optimization for different shard sizes
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
use rustfs_ecstore::api::erasure::{BitrotReader, BitrotWriter, Erasure, calc_shard_size};
use rustfs_utils::HashAlgorithm;
mod storage_api;
use std::hint::black_box;
use std::io::Cursor;
use std::time::Duration;
use storage_api::{BitrotReader, BitrotWriter, Erasure, calc_shard_size};
use tokio::runtime::Runtime;
/// Benchmark configuration structure
crates/ecstore/benches/single_block_non_inline_benchmark.rs
+2 -1
View File
@@ -13,11 +13,12 @@
// limitations under the License.
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
use rustfs_ecstore::api::erasure::{BitrotWriterWrapper, CustomWriter, Erasure};
use rustfs_utils::HashAlgorithm;
mod storage_api;
use std::io::Cursor;
use std::sync::Arc;
use std::time::Duration;
use storage_api::{BitrotWriterWrapper, CustomWriter, Erasure};
#[derive(Clone, Debug)]
struct BenchConfig {
crates/ecstore/benches/storage_api/mod.rs
+19
View File
@@ -0,0 +1,19 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
pub(crate) use rustfs_ecstore::api::erasure::{
BitrotReader, BitrotWriter, BitrotWriterWrapper, CustomWriter, Erasure, calc_shard_size,
};
crates/ecstore/src/admin_server_info.rs
+1 -1
View File
@@ -18,6 +18,7 @@ use crate::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service
use crate::{disk::endpoint::Endpoint, runtime_sources};
use crate::data_usage::load_data_usage_cache;
use crate::storage_api_contracts::StorageAdminApi;
use rustfs_common::heal_channel::DriveState;
use rustfs_madmin::{
BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, InfoMessage, ServerProperties,
@@ -26,7 +27,6 @@ use rustfs_protos::{
models::{PingBody, PingBodyBuilder},
proto_gen::node_service::{PingRequest, PingResponse},
};
use rustfs_storage_api::StorageAdminApi;
use std::{
collections::{HashMap, HashSet},
time::Duration,
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+8 -8
View File
@@ -36,6 +36,10 @@ use crate::event_notification::{EventArgs, send_event};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::runtime_sources;
use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks};
use crate::storage_api_contracts::{
DeletedObject, ExpirationOptions, HTTPRangeSpec, ListOperations as _, MultipartOperations as _, ObjectOperations as _,
ObjectToDelete,
};
use crate::store::ECStore;
use crate::tier::warm_backend::WarmBackendGetOpts;
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
@@ -57,10 +61,6 @@ use rustfs_filemeta::{
VersionPurgeStatusType, get_file_info, is_restored_object_on_disk,
};
use rustfs_s3_types::EventName;
use rustfs_storage_api::{
DeletedObject, ExpirationOptions, HTTPRangeSpec, ListOperations as _, MultipartOperations as _, ObjectOperations as _,
ObjectToDelete,
};
use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
use s3s::dto::{
BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest,
@@ -387,7 +387,7 @@ pub trait ExpiryOp: 'static {
fn as_any(&self) -> &dyn Any;
}
pub use rustfs_storage_api::TransitionedObject;
pub use crate::storage_api_contracts::TransitionedObject;
struct FreeVersionTask(ObjectInfo);
@@ -2891,13 +2891,13 @@ mod tests {
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
use crate::runtime_sources;
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
use crate::storage_api_contracts::ExpirationOptions;
use crate::storage_api_contracts::{BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations as _};
use crate::store::ECStore;
use futures::FutureExt;
use rustfs_common::metrics::{IlmAction, global_metrics};
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
use rustfs_filemeta::{ReplicateDecision, VersionPurgeStatusType};
use rustfs_storage_api::ExpirationOptions;
use rustfs_storage_api::{BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations as _};
use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, Timestamp};
use serial_test::serial;
use sha2::{Digest, Sha256};
@@ -4167,7 +4167,7 @@ mod tests {
&bucket,
object,
&upload.upload_id,
vec![rustfs_storage_api::CompletePart {
vec![crate::storage_api_contracts::CompletePart {
part_num: 1,
etag: second_part.etag.clone(),
checksum_crc32: None,
crates/ecstore/src/bucket/lifecycle/core.rs
+1 -1
View File
@@ -934,7 +934,7 @@ impl Default for Event {
}
}
pub use rustfs_storage_api::ExpirationOptions;
pub use crate::storage_api_contracts::ExpirationOptions;
#[derive(Debug, Clone)]
pub struct TransitionOptions {
crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs
+3 -1
View File
@@ -24,9 +24,11 @@ use crate::config::com::{delete_config, read_config, save_config};
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::storage_api_contracts::{
DeletedObject, HTTPRangeSpec, ListOperations as _, ObjectIO, ObjectOperations, ObjectToDelete,
};
use crate::store::ECStore;
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ListOperations as _, ObjectIO, ObjectOperations, ObjectToDelete};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs
+3 -4
View File
@@ -20,12 +20,11 @@ use tokio_util::sync::CancellationToken;
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::Result;
use crate::object_api::ObjectInfo;
use crate::storage_api_contracts::{
BucketOperations, BucketOptions, ListOperations as _, StorageObjectInfoOrErr, StorageWalkOptions,
};
use crate::store::ECStore;
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{
BucketOperations, BucketOptions, ListOperations as _, ObjectInfoOrErr as StorageObjectInfoOrErr,
WalkOptions as StorageWalkOptions,
};
pub const DEFAULT_FREE_VERSION_RECOVERY_LIMIT: usize = 1_000;
const DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT: usize = 10_000;
crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs
+1 -1
View File
@@ -23,8 +23,8 @@ use crate::bucket::lifecycle::lifecycle::{self, ObjectOpts};
use crate::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry;
use crate::client::signer_error::error_chain_contains_signer_header_marker;
use crate::runtime_sources;
use crate::storage_api_contracts::TransitionedObject;
use crate::store::ECStore;
use rustfs_storage_api::TransitionedObject;
use rustfs_utils::get_env_usize;
use sha2::{Digest, Sha256};
use std::any::Any;
crates/ecstore/src/bucket/metadata_sys.rs
+1 -1
View File
@@ -20,12 +20,12 @@ use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, load_bucket_metadata_pars
use crate::bucket::utils::{deserialize, is_meta_bucketname};
use crate::error::{Error, Result, is_err_bucket_not_found};
use crate::runtime_sources;
use crate::storage_api_contracts::HealOperations as _;
use crate::store::ECStore;
use futures::future::join_all;
use lazy_static::lazy_static;
use rustfs_common::heal_channel::HealOpts;
use rustfs_policy::policy::BucketPolicy;
use rustfs_storage_api::HealOperations as _;
use s3s::dto::ReplicationConfiguration;
use s3s::dto::{
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration,
crates/ecstore/src/bucket/migration.rs
+4 -5
View File
@@ -19,16 +19,15 @@ use crate::bucket::replication::{decode_resync_file, encode_resync_file};
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::Error;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::storage_api_contracts::{
BucketOperations, BucketOptions, DeletedObject, HTTPRangeSpec, ListOperations, ObjectIO, ObjectOperations, ObjectToDelete,
StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
};
use crate::storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations};
use http::HeaderMap;
use rustfs_filemeta::FileInfo;
use rustfs_policy::auth::UserIdentity;
use rustfs_policy::policy::PolicyDoc;
use rustfs_storage_api::{
BucketOperations, BucketOptions, DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectIO, ObjectInfoOrErr as StorageObjectInfoOrErr,
ObjectOperations, ObjectToDelete, WalkOptions as StorageWalkOptions,
};
use rustfs_utils::path::SLASH_SEPARATOR;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
crates/ecstore/src/bucket/replication/replication_pool.rs
+1 -1
View File
@@ -29,6 +29,7 @@ use crate::disk::BUCKET_META_PREFIX;
use crate::error::Error as EcstoreError;
use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::runtime_sources;
use crate::storage_api_contracts::DeletedObject;
use crate::storage_api_contracts::EcstoreObjectIO;
use lazy_static::lazy_static;
use rustfs_filemeta::MrfOpKind;
@@ -44,7 +45,6 @@ use rustfs_filemeta::VersionPurgeStatusType;
use rustfs_filemeta::replication_statuses_map;
use rustfs_filemeta::version_purge_statuses_map;
use rustfs_filemeta::{REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE};
use rustfs_storage_api::DeletedObject;
use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str};
use std::any::Any;
use std::sync::Arc;
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+4 -5
View File
@@ -31,6 +31,10 @@ use crate::event_notification::{EventArgs, send_event};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::runtime_sources;
use crate::set_disk::get_lock_acquire_timeout;
use crate::storage_api_contracts::{
DeletedObject, HTTPRangeSpec, ListOperations, NamespaceLocking as StorageNamespaceLocking, ObjectIO, ObjectOperations,
ObjectToDelete, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
};
use crate::storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations};
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput};
@@ -56,11 +60,6 @@ use rustfs_filemeta::{
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
};
use rustfs_s3_types::EventName;
use rustfs_storage_api::{
DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, NamespaceLocking as StorageNamespaceLocking, ObjectIO,
ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations, ObjectToDelete, WalkOptions as StorageWalkOptions,
};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
crates/ecstore/src/client/api_list.rs
+1 -1
View File
@@ -26,12 +26,12 @@ use crate::client::{
credentials,
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
};
use crate::storage_api_contracts::BucketInfo;
use http::{HeaderMap, StatusCode};
use http_body_util::BodyExt;
use hyper::body::Body;
use hyper::body::Bytes;
use rustfs_config::MAX_S3_CLIENT_RESPONSE_SIZE;
use rustfs_storage_api::BucketInfo;
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
use std::collections::HashMap;
use std::io::ErrorKind;
crates/ecstore/src/client/object_api_utils.rs
+1 -1
View File
@@ -26,9 +26,9 @@ use tokio::io::BufReader;
use crate::error::ErrorResponse;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::storage_api_contracts::HTTPRangeSpec;
use rustfs_filemeta::ObjectPartInfo;
use rustfs_rio::HashReader;
use rustfs_storage_api::HTTPRangeSpec;
use s3s::S3ErrorCode;
//#[derive(Clone)]
crates/ecstore/src/client/object_handlers_common.rs
+1 -1
View File
@@ -25,10 +25,10 @@ use crate::bucket::replication::{DeletedObjectReplicationInfo, check_replicate_d
use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::object_api::ObjectOptions;
use crate::storage_api_contracts::{ObjectOperations as _, ObjectToDelete};
use crate::store::ECStore;
use rustfs_filemeta::{REPLICATE_INCOMING_DELETE, ReplicationState, version_purge_statuses_map};
use rustfs_lock::MAX_DELETE_LIST;
use rustfs_storage_api::{ObjectOperations as _, ObjectToDelete};
fn lifecycle_version_delete_replication_state(
replicate_decision_str: String,
crates/ecstore/src/cluster/mod.rs
+1 -1
View File
@@ -14,7 +14,7 @@
use std::collections::{BTreeMap, BTreeSet};
use rustfs_storage_api::{
use crate::storage_api_contracts::{
CapabilityStatus, DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet,
TopologySnapshot,
};
crates/ecstore/src/config/com.rs
+6 -6
View File
@@ -18,6 +18,7 @@ use crate::error::{Error, Result};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::runtime_sources;
use crate::storage_api_contracts::EcstoreObjectIO;
use crate::storage_api_contracts::{DeletedObject, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete, StorageAdminApi};
use http::HeaderMap;
use rustfs_config::audit::{
AUDIT_AMQP_KEYS, AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS,
@@ -34,7 +35,6 @@ use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_RE
use rustfs_config::server_config::{Config, KVS};
use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete, StorageAdminApi};
use rustfs_utils::path::SLASH_SEPARATOR;
use serde_json::{Map, Value};
use std::collections::{HashMap, HashSet};
@@ -1315,6 +1315,8 @@ mod tests {
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::runtime_sources;
use crate::set_disk::SetDisks;
use crate::storage_api_contracts::NamespaceLocking as _;
use crate::storage_api_contracts::{HTTPRangeSpec, StorageAdminApi};
use http::HeaderMap;
use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{
@@ -1328,8 +1330,6 @@ mod tests {
use rustfs_lock::client::LockClient;
use rustfs_lock::client::local::LocalClient;
use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};
use rustfs_storage_api::NamespaceLocking as _;
use rustfs_storage_api::{HTTPRangeSpec, StorageAdminApi};
use serde_json::Value;
use serial_test::serial;
use std::collections::HashMap;
@@ -1498,7 +1498,7 @@ mod tests {
}
#[async_trait::async_trait]
impl rustfs_storage_api::ObjectIO for LockingConfigStorage {
impl crate::storage_api_contracts::ObjectIO for LockingConfigStorage {
type Error = Error;
type RangeSpec = HTTPRangeSpec;
type HeaderMap = HeaderMap;
@@ -1549,7 +1549,7 @@ mod tests {
}
#[async_trait::async_trait]
impl rustfs_storage_api::NamespaceLocking for LockingConfigStorage {
impl crate::storage_api_contracts::NamespaceLocking for LockingConfigStorage {
type Error = crate::error::Error;
type NamespaceLock = rustfs_lock::NamespaceLockWrapper;
@@ -1579,7 +1579,7 @@ mod tests {
async fn disk_set_inventory(
&self,
_selector: rustfs_storage_api::DiskSetSelector,
_selector: crate::storage_api_contracts::DiskSetSelector,
) -> Result<Vec<Option<crate::disk::DiskStore>>> {
panic!("unused in test")
}
crates/ecstore/src/data_movement.rs
+1 -1
View File
@@ -15,11 +15,11 @@
use crate::error::{Error, Result, is_err_data_movement_overwrite, is_err_object_not_found, is_err_version_not_found};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::set_disk::SetDisks;
use crate::storage_api_contracts::{CompletePart, MultipartOperations as _, ObjectIO as _, ObjectOperations as _};
use crate::store::ECStore;
use bytes::Bytes;
use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
use rustfs_rio::{ChecksumType, EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
use rustfs_storage_api::{CompletePart, MultipartOperations as _, ObjectIO as _, ObjectOperations as _};
use rustfs_utils::http::AMZ_OBJECT_TAGGING;
use rustfs_utils::path::encode_dir_object;
use std::collections::{BTreeMap, HashMap};
crates/ecstore/src/data_usage.rs
+6 -2
View File
@@ -14,6 +14,7 @@
pub mod local_snapshot;
use crate::storage_api_contracts::{ListOperations as _, ObjectIO as _};
use crate::{
bucket::metadata_sys::get_replication_config,
config::com::read_config,
@@ -27,7 +28,6 @@ use rustfs_data_usage::{
BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary,
};
use rustfs_io_metrics::record_system_path_failure;
use rustfs_storage_api::{ListOperations as _, ObjectIO as _};
use rustfs_utils::path::SLASH_SEPARATOR;
use std::{
collections::{HashMap, HashSet, hash_map::Entry},
@@ -746,7 +746,11 @@ pub fn create_cache_entry_from_summary(summary: &SizeSummary) -> DataUsageEntry
}
/// Convert data usage cache to DataUsageInfo
pub fn cache_to_data_usage_info(cache: &DataUsageCache, path: &str, buckets: &[rustfs_storage_api::BucketInfo]) -> DataUsageInfo {
pub fn cache_to_data_usage_info(
cache: &DataUsageCache,
path: &str,
buckets: &[crate::storage_api_contracts::BucketInfo],
) -> DataUsageInfo {
let e = match cache.find(path) {
Some(e) => e,
None => return DataUsageInfo::default(),
crates/ecstore/src/error.rs
+1 -1
View File
@@ -14,7 +14,7 @@
use crate::bucket::error::BucketMetadataError;
use crate::disk::error::DiskError;
use rustfs_storage_api::{HTTPRangeError, StorageErrorCode};
use crate::storage_api_contracts::{HTTPRangeError, StorageErrorCode};
use rustfs_utils::path::decode_dir_object;
use s3s::{S3Error, S3ErrorCode};
crates/ecstore/src/metrics_realtime.rs
+1 -1
View File
@@ -14,6 +14,7 @@
use crate::admin_server_info::get_local_server_property;
use crate::runtime_sources;
use crate::storage_api_contracts::StorageAdminApi;
use chrono::Utc;
use rustfs_common::{heal_channel::DriveState, metrics::global_metrics};
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
@@ -29,7 +30,6 @@ use rustfs_madmin::metrics::{
ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, ScannerSourceWorkSnapshot as MadminScannerSourceWorkSnapshot,
ScannerUsageFreshnessSnapshot as MadminScannerUsageFreshnessSnapshot, TimedAction as MadminTimedAction,
};
use rustfs_storage_api::StorageAdminApi;
use rustfs_utils::os::get_drive_stats;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
crates/ecstore/src/notification_sys.rs
+1 -1
View File
@@ -19,13 +19,13 @@ use crate::metrics_realtime::{CollectMetricsOpts, MetricType};
use crate::rebalance::RebalSaveOpt;
use crate::rpc::PeerRestClient;
use crate::runtime_sources;
use crate::storage_api_contracts::StorageAdminApi;
use futures::future::join_all;
use lazy_static::lazy_static;
use rustfs_madmin::health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysServices};
use rustfs_madmin::metrics::RealtimeMetrics;
use rustfs_madmin::net::NetInfo;
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
use rustfs_storage_api::StorageAdminApi;
use std::collections::hash_map::DefaultHasher;
use std::future::Future;
use std::hash::{Hash, Hasher};
crates/ecstore/src/object_api/mod.rs
+1 -1
View File
@@ -17,6 +17,7 @@ use crate::bucket::versioning::VersioningApi as _;
use crate::config::storageclass;
use crate::error::{Error, Result};
use crate::rio::{HashReader, LimitReader};
use crate::storage_api_contracts::{ExpirationOptions, HTTPRangeSpec, TransitionedObject};
use crate::store_utils::clean_metadata;
use crate::{bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, bucket::lifecycle::lifecycle::TransitionOptions};
use bytes::Bytes;
@@ -27,7 +28,6 @@ use rustfs_filemeta::{
version_purge_statuses_map,
};
use rustfs_rio::Checksum;
use rustfs_storage_api::{ExpirationOptions, HTTPRangeSpec, TransitionedObject};
use rustfs_utils::CompressionAlgorithm;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS};
crates/ecstore/src/object_api/types.rs
+1 -1
View File
@@ -1,5 +1,5 @@
use super::*;
use rustfs_storage_api::{
use crate::storage_api_contracts::{
HTTPPreconditions, ObjectLockRetentionOptions, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState,
VersionMarker,
};
crates/ecstore/src/pools.rs
+4 -4
View File
@@ -39,6 +39,10 @@ use crate::object_api::{GetObjectReader, ObjectOptions};
use crate::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission};
use crate::runtime_sources;
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
use crate::storage_api_contracts::{
BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, NamespaceLocking as _, ObjectIO as _,
ObjectOperations as _, StorageAdminApi,
};
use crate::{sets::Sets, store::ECStore};
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered};
@@ -49,10 +53,6 @@ use rmp_serde::Serializer;
use rustfs_common::defer;
use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_storage_api::{
BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, NamespaceLocking as _, ObjectIO as _,
ObjectOperations as _, StorageAdminApi,
};
use rustfs_utils::path::{encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path};
use s3s::dto::{BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration};
use serde::{Deserialize, Serialize};
crates/ecstore/src/rebalance/control.rs
+1 -1
View File
@@ -20,9 +20,9 @@ use crate::error::{Error, Result};
use crate::object_api::ObjectOptions;
use crate::set_disk::get_lock_acquire_timeout;
use crate::storage_api_contracts::EcstoreObjectIO;
use crate::storage_api_contracts::{NamespaceLocking as StorageNamespaceLocking, StorageAdminApi};
use crate::store::ECStore;
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{NamespaceLocking as StorageNamespaceLocking, StorageAdminApi};
use std::sync::Arc;
use time::OffsetDateTime;
use tracing::{debug, info};
crates/ecstore/src/rebalance/entry.rs
+1 -1
View File
@@ -34,9 +34,9 @@ use crate::error::{Error, Result};
use crate::object_api::{GetObjectReader, ObjectOptions};
use crate::pools::ListCallback;
use crate::set_disk::SetDisks;
use crate::storage_api_contracts::ObjectOperations as _;
use crate::store::ECStore;
use rustfs_filemeta::MetaCacheEntry;
use rustfs_storage_api::ObjectOperations as _;
use rustfs_utils::path::encode_dir_object;
use std::sync::Arc;
use time::OffsetDateTime;
crates/ecstore/src/rebalance/meta.rs
+1 -1
View File
@@ -7,9 +7,9 @@ use super::{
};
use crate::config::com::{read_config_with_metadata, save_config_with_opts};
use crate::error::is_err_operation_canceled;
use crate::storage_api_contracts::{HTTPRangeSpec, ObjectIO};
use http::HeaderMap;
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO};
use std::{collections::HashSet, fmt, io::Cursor, sync::Arc};
use time::OffsetDateTime;
use tracing::{debug, info};
crates/ecstore/src/rebalance/migration.rs
+1 -1
View File
@@ -3,9 +3,9 @@ use crate::data_usage::DATA_USAGE_CACHE_NAME;
use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_found};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::set_disk::SetDisks;
use crate::storage_api_contracts::{HTTPRangeSpec, ObjectIO, ObjectOperations as _};
use http::HeaderMap;
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO, ObjectOperations as _};
use rustfs_utils::path::encode_dir_object;
use std::future::Future;
use tokio::time::Duration;
crates/ecstore/src/rebalance/rebalance_unit_tests.rs
+1 -1
View File
@@ -54,10 +54,10 @@ use crate::data_usage::DATA_USAGE_CACHE_NAME;
use crate::disk::RUSTFS_META_BUCKET;
use crate::disk::error::DiskError;
use crate::error::{Error, Result};
use crate::storage_api_contracts::HTTPRangeSpec;
use rustfs_filemeta::FileInfo;
use rustfs_filemeta::TRANSITION_COMPLETE;
use rustfs_rio::Index;
use rustfs_storage_api::HTTPRangeSpec;
use s3s::dto::ReplicationConfiguration;
use serde::Serialize;
use std::io::Cursor;
crates/ecstore/src/rpc/peer_s3_client.rs
+1 -1
View File
@@ -21,6 +21,7 @@ use crate::rpc::client::{
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
};
use crate::runtime_sources;
use crate::storage_api_contracts::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions};
use crate::store::all_local_disk;
use crate::store_utils::is_reserved_or_invalid_bucket;
use crate::{
@@ -38,7 +39,6 @@ use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClie
use rustfs_protos::proto_gen::node_service::{
DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest,
};
use rustfs_storage_api::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions};
use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration};
use tokio::{net::TcpStream, sync::RwLock, time};
use tokio_util::sync::CancellationToken;
crates/ecstore/src/set_disk.rs
+29 -23
View File
@@ -42,6 +42,14 @@ use crate::get_diagnostics::{
use crate::object_api::ObjectOptions;
use crate::rpc::heal_bucket_local_on_disks;
use crate::runtime_sources;
use crate::storage_api_contracts::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
};
use crate::storage_api_contracts::{
HTTPRangeSpec, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
};
use crate::storage_api_contracts::{MultipartOperations as _, NamespaceLocking as _, ObjectIO as _, ObjectOperations as _};
use crate::store_utils::is_reserved_or_invalid_bucket;
use crate::{
bucket::lifecycle::bucket_lifecycle_ops::{
@@ -88,15 +96,6 @@ use rustfs_object_capacity::capacity_scope::{
CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope,
};
use rustfs_s3_types::EventName;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
};
use rustfs_storage_api::{
HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
};
use rustfs_storage_api::{MultipartOperations as _, NamespaceLocking as _, ObjectIO as _, ObjectOperations as _};
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
use rustfs_utils::http::headers::{
@@ -1023,7 +1022,7 @@ fn classify_multipart_part_write_path(object_size: i64, block_size: usize) -> Sm
}
#[async_trait::async_trait]
impl rustfs_storage_api::ObjectIO for SetDisks {
impl crate::storage_api_contracts::ObjectIO for SetDisks {
type Error = Error;
type RangeSpec = HTTPRangeSpec;
type HeaderMap = HeaderMap;
@@ -2011,7 +2010,7 @@ impl SetDisks {
}
#[async_trait::async_trait]
impl rustfs_storage_api::NamespaceLocking for SetDisks {
impl crate::storage_api_contracts::NamespaceLocking for SetDisks {
type Error = Error;
type NamespaceLock = NamespaceLockWrapper;
@@ -2267,7 +2266,7 @@ fn check_object_lock_retention_update(bucket: &str, object: &str, obj_info: &Obj
}
#[async_trait::async_trait]
impl rustfs_storage_api::ObjectOperations for SetDisks {
impl crate::storage_api_contracts::ObjectOperations for SetDisks {
type Error = Error;
type ObjectInfo = ObjectInfo;
type ObjectOptions = ObjectOptions;
@@ -3373,8 +3372,15 @@ impl rustfs_storage_api::ObjectOperations for SetDisks {
#[tracing::instrument(skip(self))]
async fn verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
let get_object_reader =
<Self as rustfs_storage_api::ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
let get_object_reader = <Self as crate::storage_api_contracts::ObjectIO>::get_object_reader(
self,
bucket,
object,
None,
HeaderMap::new(),
opts,
)
.await?;
// Stream to sink to avoid loading entire object into memory during verification
let mut reader = get_object_reader.stream;
tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
@@ -3497,7 +3503,7 @@ impl SetDisks {
}
#[async_trait::async_trait]
impl rustfs_storage_api::ListOperations for SetDisks {
impl crate::storage_api_contracts::ListOperations for SetDisks {
type Error = Error;
type ListObjectsV2Info = ListObjectsV2Info;
type ListObjectVersionsInfo = ListObjectVersionsInfo;
@@ -3558,7 +3564,7 @@ impl rustfs_storage_api::ListOperations for SetDisks {
}
#[async_trait::async_trait]
impl rustfs_storage_api::MultipartOperations for SetDisks {
impl crate::storage_api_contracts::MultipartOperations for SetDisks {
type Error = Error;
type ObjectInfo = ObjectInfo;
type ObjectOptions = ObjectOptions;
@@ -4666,7 +4672,7 @@ impl rustfs_storage_api::MultipartOperations for SetDisks {
}
#[async_trait::async_trait]
impl rustfs_storage_api::HealOperations for SetDisks {
impl crate::storage_api_contracts::HealOperations for SetDisks {
type Error = Error;
type HealResultItem = HealResultItem;
type HealOptions = HealOpts;
@@ -5621,6 +5627,10 @@ mod tests {
use crate::disk::health_state::RuntimeDriveHealthState;
use crate::endpoints::SetupType;
use crate::object_api::ObjectInfo;
use crate::storage_api_contracts::HealOperations as _;
use crate::storage_api_contracts::ListOperations as _;
use crate::storage_api_contracts::TransitionedObject;
use crate::storage_api_contracts::{CompletePart, NamespaceLocking as _, ObjectOperations as _};
use crate::store_init::save_format_file;
use crate::store_list_objects::ListPathOptions;
use rustfs_filemeta::ErasureInfo;
@@ -5628,10 +5638,6 @@ mod tests {
use rustfs_filemeta::ReplicationState;
use rustfs_lock::client::local::LocalClient;
use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};
use rustfs_storage_api::HealOperations as _;
use rustfs_storage_api::ListOperations as _;
use rustfs_storage_api::TransitionedObject;
use rustfs_storage_api::{CompletePart, NamespaceLocking as _, ObjectOperations as _};
use serial_test::serial;
use std::collections::HashMap;
use tempfile::TempDir;
@@ -7442,7 +7448,7 @@ mod tests {
..Default::default()
};
let opts = ObjectOptions {
object_lock_retention: Some(rustfs_storage_api::ObjectLockRetentionOptions {
object_lock_retention: Some(crate::storage_api_contracts::ObjectLockRetentionOptions {
mode: Some(s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string()),
retain_until: Some(requested_until),
bypass_governance: true,
@@ -7477,7 +7483,7 @@ mod tests {
..Default::default()
};
let opts = ObjectOptions {
object_lock_retention: Some(rustfs_storage_api::ObjectLockRetentionOptions {
object_lock_retention: Some(crate::storage_api_contracts::ObjectLockRetentionOptions {
mode: Some(s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string()),
retain_until: Some(requested_until),
bypass_governance: true,
crates/ecstore/src/set_disk/heal.rs
+1 -1
View File
@@ -13,8 +13,8 @@
// limitations under the License.
use super::*;
use crate::storage_api_contracts::NamespaceLocking as _;
use rustfs_config::{DEFAULT_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE};
use rustfs_storage_api::NamespaceLocking as _;
impl SetDisks {
#[tracing::instrument(skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
crates/ecstore/src/sets.rs
+16 -17
View File
@@ -16,6 +16,14 @@
use crate::disk::error_reduce::count_errs;
use crate::error::{Error, Result};
use crate::layout::set_heal::{formats_to_drives_info, new_heal_format_sets};
use crate::storage_api_contracts::CompletePart;
use crate::storage_api_contracts::HTTPRangeSpec;
use crate::storage_api_contracts::{
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DeletedObject, ListMultipartsInfo, ListPartsInfo,
MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo, StorageListObjectVersionsInfo,
StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
};
use crate::storage_api_contracts::{ObjectIO as _, ObjectOperations as _};
use crate::{
disk::{
DiskAPI, DiskOption, DiskStore,
@@ -41,15 +49,6 @@ use rustfs_filemeta::FileInfo;
use rustfs_lock::NamespaceLockWrapper;
use rustfs_lock::client::LockClient;
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_storage_api::CompletePart;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListPartsInfo,
MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectToDelete, PartInfo,
WalkOptions as StorageWalkOptions,
};
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
use std::{collections::HashMap, sync::Arc};
use tokio::sync::RwLock;
@@ -401,7 +400,7 @@ fn apply_delete_objects_results(
}
#[async_trait::async_trait]
impl rustfs_storage_api::ObjectIO for Sets {
impl crate::storage_api_contracts::ObjectIO for Sets {
type Error = Error;
type RangeSpec = HTTPRangeSpec;
type HeaderMap = HeaderMap;
@@ -494,7 +493,7 @@ impl BucketOperations for Sets {
}
#[async_trait::async_trait]
impl rustfs_storage_api::ObjectOperations for Sets {
impl crate::storage_api_contracts::ObjectOperations for Sets {
type Error = Error;
type ObjectInfo = ObjectInfo;
type ObjectOptions = ObjectOptions;
@@ -702,7 +701,7 @@ impl rustfs_storage_api::ObjectOperations for Sets {
}
#[async_trait::async_trait]
impl rustfs_storage_api::ListOperations for Sets {
impl crate::storage_api_contracts::ListOperations for Sets {
type Error = Error;
type ListObjectsV2Info = ListObjectsV2Info;
type ListObjectVersionsInfo = ListObjectVersionsInfo;
@@ -763,7 +762,7 @@ impl rustfs_storage_api::ListOperations for Sets {
}
#[async_trait::async_trait]
impl rustfs_storage_api::MultipartOperations for Sets {
impl crate::storage_api_contracts::MultipartOperations for Sets {
type Error = Error;
type ObjectInfo = ObjectInfo;
type ObjectOptions = ObjectOptions;
@@ -877,7 +876,7 @@ impl rustfs_storage_api::MultipartOperations for Sets {
}
#[async_trait::async_trait]
impl rustfs_storage_api::HealOperations for Sets {
impl crate::storage_api_contracts::HealOperations for Sets {
type Error = Error;
type HealResultItem = HealResultItem;
type HealOptions = HealOpts;
@@ -1017,7 +1016,7 @@ impl rustfs_storage_api::HealOperations for Sets {
}
#[async_trait::async_trait]
impl rustfs_storage_api::NamespaceLocking for Sets {
impl crate::storage_api_contracts::NamespaceLocking for Sets {
type Error = Error;
type NamespaceLock = NamespaceLockWrapper;
@@ -1093,8 +1092,8 @@ async fn init_storage_disks_with_errors(
mod tests {
use super::*;
use crate::layout::endpoint::Endpoint;
use rustfs_storage_api::HealOperations as _;
use rustfs_storage_api::ListOperations as _;
use crate::storage_api_contracts::HealOperations as _;
use crate::storage_api_contracts::ListOperations as _;
#[test]
fn test_apply_delete_objects_results_preserves_original_order_for_out_of_order_batches() {
crates/ecstore/src/storage_api_contracts.rs
+16 -10
View File
@@ -1,10 +1,16 @@
use crate::error::Error;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{
DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectToDelete,
WalkOptions as StorageWalkOptions,
pub use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, CapabilityStatus, CompletePart, DeleteBucketOptions, DeletedObject,
DiskCapabilities, DiskSetSelector, ExpirationOptions, HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, HealOperations,
ListMultipartsInfo, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ListPartsInfo, MakeBucketOptions, MultipartInfo,
MultipartOperations, MultipartUploadResult, NamespaceLocking, ObjectIO, ObjectInfoOrErr as StorageObjectInfoOrErr,
ObjectLockRetentionOptions, ObjectOperations, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState,
ObjectToDelete, PartInfo, StorageAdminApi, StorageErrorCode, TopologyCapabilities, TopologyDisk, TopologyLabels,
TopologyPool, TopologySet, TopologySnapshot, TransitionedObject, VersionMarker, WalkOptions as StorageWalkOptions,
WalkVersionsSortOrder,
};
use std::fmt::Debug;
use tokio_util::sync::CancellationToken;
@@ -15,7 +21,7 @@ type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
pub(crate) trait EcstoreObjectIO:
rustfs_storage_api::ObjectIO<
ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
@@ -31,7 +37,7 @@ pub(crate) trait EcstoreObjectIO:
}
impl<T> EcstoreObjectIO for T where
T: rustfs_storage_api::ObjectIO<
T: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
@@ -47,7 +53,7 @@ impl<T> EcstoreObjectIO for T where
}
pub(crate) trait EcstoreObjectOperations:
rustfs_storage_api::ObjectOperations<
ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
@@ -61,7 +67,7 @@ pub(crate) trait EcstoreObjectOperations:
}
impl<T> EcstoreObjectOperations for T where
T: rustfs_storage_api::ObjectOperations<
T: ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
@@ -75,7 +81,7 @@ impl<T> EcstoreObjectOperations for T where
}
pub(crate) trait EcstoreListOperations:
rustfs_storage_api::ListOperations<
ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
@@ -90,7 +96,7 @@ pub(crate) trait EcstoreListOperations:
}
impl<T> EcstoreListOperations for T where
T: rustfs_storage_api::ListOperations<
T: ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
crates/ecstore/src/store.rs
+18 -16
View File
@@ -46,6 +46,13 @@ use crate::pools::PoolMeta;
use crate::rebalance::RebalanceMeta;
use crate::rpc::RemoteClient;
use crate::runtime_sources;
use crate::storage_api_contracts::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
};
use crate::storage_api_contracts::{
HTTPRangeSpec, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions,
};
use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
use crate::tier::tier::TierConfigMgr;
use crate::{
@@ -66,14 +73,6 @@ use rustfs_config::server_config::Config;
use rustfs_filemeta::FileInfo;
use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
};
use rustfs_storage_api::{
HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
};
use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf};
use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
use std::net::SocketAddr;
@@ -347,7 +346,7 @@ impl ECStore {
// }
#[async_trait::async_trait]
impl rustfs_storage_api::ObjectIO for ECStore {
impl crate::storage_api_contracts::ObjectIO for ECStore {
type Error = Error;
type RangeSpec = HTTPRangeSpec;
type HeaderMap = HeaderMap;
@@ -408,7 +407,7 @@ impl BucketOperations for ECStore {
}
#[async_trait::async_trait]
impl rustfs_storage_api::ObjectOperations for ECStore {
impl crate::storage_api_contracts::ObjectOperations for ECStore {
type Error = Error;
type ObjectInfo = ObjectInfo;
type ObjectOptions = ObjectOptions;
@@ -499,7 +498,7 @@ impl rustfs_storage_api::ObjectOperations for ECStore {
}
#[async_trait::async_trait]
impl rustfs_storage_api::ListOperations for ECStore {
impl crate::storage_api_contracts::ListOperations for ECStore {
type Error = Error;
type ListObjectsV2Info = ListObjectsV2Info;
type ListObjectVersionsInfo = ListObjectVersionsInfo;
@@ -564,7 +563,7 @@ impl rustfs_storage_api::ListOperations for ECStore {
}
#[async_trait::async_trait]
impl rustfs_storage_api::MultipartOperations for ECStore {
impl crate::storage_api_contracts::MultipartOperations for ECStore {
type Error = Error;
type ObjectInfo = ObjectInfo;
type ObjectOptions = ObjectOptions;
@@ -688,7 +687,7 @@ impl rustfs_storage_api::MultipartOperations for ECStore {
}
#[async_trait::async_trait]
impl rustfs_storage_api::HealOperations for ECStore {
impl crate::storage_api_contracts::HealOperations for ECStore {
type Error = Error;
type HealResultItem = HealResultItem;
type HealOptions = HealOpts;
@@ -725,7 +724,7 @@ impl rustfs_storage_api::HealOperations for ECStore {
}
#[async_trait::async_trait]
impl rustfs_storage_api::NamespaceLocking for ECStore {
impl crate::storage_api_contracts::NamespaceLocking for ECStore {
type Error = Error;
type NamespaceLock = NamespaceLockWrapper;
@@ -735,7 +734,7 @@ impl rustfs_storage_api::NamespaceLocking for ECStore {
}
#[async_trait::async_trait]
impl rustfs_storage_api::StorageAdminApi for ECStore {
impl crate::storage_api_contracts::StorageAdminApi for ECStore {
type BackendInfo = rustfs_madmin::BackendInfo;
type StorageInfo = rustfs_madmin::StorageInfo;
type Disk = DiskStore;
@@ -757,7 +756,10 @@ impl rustfs_storage_api::StorageAdminApi for ECStore {
}
#[instrument(skip(self))]
async fn disk_set_inventory(&self, selector: rustfs_storage_api::DiskSetSelector) -> Result<Vec<Option<Self::Disk>>> {
async fn disk_set_inventory(
&self,
selector: crate::storage_api_contracts::DiskSetSelector,
) -> Result<Vec<Option<Self::Disk>>> {
self.handle_get_disks(selector.pool_idx, selector.set_idx).await
}
crates/ecstore/src/store/bucket.rs
+1 -1
View File
@@ -19,7 +19,7 @@ use crate::bucket::{
};
use crate::runtime_sources;
use crate::set_disk::get_lock_acquire_timeout;
use rustfs_storage_api::NamespaceLocking as _;
use crate::storage_api_contracts::NamespaceLocking as _;
fn should_override_created_from_metadata(created: OffsetDateTime) -> bool {
created != OffsetDateTime::UNIX_EPOCH
crates/ecstore/src/store/heal.rs
+1 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use super::*;
use rustfs_storage_api::HealOperations as _;
use crate::storage_api_contracts::HealOperations as _;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_HEAL: &str = "heal";
crates/ecstore/src/store/multipart.rs
+1 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use super::*;
use rustfs_storage_api::MultipartOperations as _;
use crate::storage_api_contracts::MultipartOperations as _;
impl ECStore {
#[instrument(skip(self))]
crates/ecstore/src/store/object.rs
+10 -3
View File
@@ -17,11 +17,11 @@ use crate::set_disk::{
get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
is_lock_optimization_enabled, is_object_lock_diag_enabled,
};
use crate::storage_api_contracts::{ObjectIO as _, ObjectOperations as _};
use rustfs_io_metrics::{
record_object_lock_diag_acquire_duration, record_object_lock_diag_hold_duration, record_object_lock_diag_slow_acquire,
record_object_lock_diag_slow_hold,
};
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
use std::{
fmt,
pin::Pin,
@@ -1281,8 +1281,15 @@ impl ECStore {
}
pub(super) async fn handle_verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
let get_object_reader =
<Self as rustfs_storage_api::ObjectIO>::get_object_reader(self, bucket, object, None, HeaderMap::new(), opts).await?;
let get_object_reader = <Self as crate::storage_api_contracts::ObjectIO>::get_object_reader(
self,
bucket,
object,
None,
HeaderMap::new(),
opts,
)
.await?;
// Stream to sink to avoid loading entire object into memory during verification
let mut reader = get_object_reader.stream;
tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
crates/ecstore/src/store/rebalance.rs
+1 -1
View File
@@ -15,7 +15,7 @@
use super::*;
use crate::layout::pool_space::{ServerPoolsAvailableSpace, build_server_pools_available_space};
use crate::runtime_sources;
use rustfs_storage_api::{NamespaceLocking as _, ObjectOperations as _, StorageAdminApi};
use crate::storage_api_contracts::{NamespaceLocking as _, ObjectOperations as _, StorageAdminApi};
pub(in crate::store) mod support;
use support::{
LatestObjectInfoCandidate, PoolErr, PoolObjInfo, RebalanceDeletePoolResult, pool_lookup_not_found_error,
crates/ecstore/src/store_list_objects.rs
+4 -5
View File
@@ -24,6 +24,10 @@ use crate::error::{
use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::set_disk::SetDisks;
use crate::sets::Sets;
use crate::storage_api_contracts::{
ListObjectsInfo as StorageListObjectsInfo, ObjectOperations as _, StorageListObjectVersionsInfo, StorageListObjectsV2Info,
StorageObjectInfoOrErr, StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
};
use crate::store::ECStore;
use crate::store_utils::is_reserved_or_invalid_bucket;
use futures::future::join_all;
@@ -32,11 +36,6 @@ use rustfs_filemeta::{
MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
merge_file_meta_versions,
};
use rustfs_storage_api::{
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo as StorageListObjectsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations as _,
VersionMarker, WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
};
use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
crates/ecstore/src/tier/tier.rs
+1 -1
View File
@@ -38,6 +38,7 @@ use tracing::{debug, error, info, warn};
use crate::client::admin_handler_utils::AdminError;
use crate::error::{Error, Result, StorageError};
use crate::storage_api_contracts::{DeletedObject, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete};
use crate::tier::{
tier_admin::TierCreds,
tier_config::{TierConfig, TierType},
@@ -54,7 +55,6 @@ use crate::{
};
use rustfs_filemeta::FileInfo;
use rustfs_rio::HashReader;
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::S3ErrorCode;
crates/ecstore/tests/ecstore_contract_compat_test.rs
+9 -9
View File
@@ -12,18 +12,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
mod storage_api;
use rustfs_common::heal_channel::HealOpts;
use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use rustfs_ecstore::api::{disk::DiskStore, error::Error, storage::ECStore};
use rustfs_filemeta::FileInfo;
use rustfs_lock::NamespaceLockWrapper;
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::{
CompletePart, DeletedObject, HTTPRangeSpec, HealOperations as StorageHealOperations, ListMultipartsInfo,
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListPartsInfo,
MultipartInfo, MultipartOperations as StorageMultipartOperations, MultipartUploadResult,
NamespaceLocking as StorageNamespaceLocking, ObjectIO as StorageObjectIO, ObjectInfoOrErr as StorageObjectInfoOrErr,
ObjectOperations as StorageObjectOperations, ObjectToDelete, PartInfo, StorageAdminApi, WalkOptions as StorageWalkOptions,
use storage_api::{
CompletePart, DeletedObject, DiskStore, ECStore, Error, GetObjectReader, HTTPRangeSpec, ListMultipartsInfo, ListPartsInfo,
MultipartInfo, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAdminApi,
StorageHealOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageListOperations,
StorageMultipartOperations, StorageNamespaceLocking, StorageObjectIO, StorageObjectInfoOrErr, StorageObjectOperations,
StorageWalkOptions,
};
use tokio_util::sync::CancellationToken;
@@ -82,7 +82,7 @@ where
fn storage_list_operations_type_name<T>() -> &'static str
where
T: rustfs_storage_api::ListOperations<
T: StorageListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
crates/ecstore/tests/legacy_bitrot_read_test.rs
+3 -3
View File
@@ -27,12 +27,12 @@
//!
//! For MinIO data: RUSTFS_LEGACY_TEST_ROOT=/path/to/minio (disk "test" and .minio.sys auto-detected).
use rustfs_ecstore::api::bitrot::create_bitrot_reader;
use rustfs_ecstore::api::disk::endpoint::Endpoint;
use rustfs_ecstore::api::disk::{DiskOption, STORAGE_FORMAT_FILE, new_disk};
mod storage_api;
use rustfs_filemeta::{FileInfoOpts, get_file_info};
use rustfs_utils::HashAlgorithm;
use std::path::PathBuf;
use storage_api::{DiskOption, Endpoint, STORAGE_FORMAT_FILE, create_bitrot_reader, new_disk};
use tokio::fs;
fn workspace_root() -> PathBuf {
crates/ecstore/tests/minio_generated_read_test.rs
+5 -5
View File
@@ -4,14 +4,14 @@ use std::fs;
use std::io::Cursor;
use std::path::{Path, PathBuf};
use rustfs_ecstore::api::bitrot::create_bitrot_reader;
use rustfs_ecstore::api::disk::endpoint::Endpoint;
use rustfs_ecstore::api::disk::{DiskAPI as _, DiskOption, new_disk};
use rustfs_ecstore::api::erasure::Erasure;
use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions};
mod storage_api;
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use storage_api::{
DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
};
use temp_env::async_with_vars;
use tokio::io::{AsyncReadExt, AsyncWrite};
crates/ecstore/tests/storage_api.rs
+15
View File
@@ -0,0 +1,15 @@
#![allow(unused_imports)]
pub(crate) use rustfs_ecstore::api::bitrot::create_bitrot_reader;
pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, DiskStore, STORAGE_FORMAT_FILE, endpoint::Endpoint, new_disk};
pub(crate) use rustfs_ecstore::api::erasure::Erasure;
pub(crate) use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
pub(crate) use rustfs_ecstore::api::{error::Error, storage::ECStore};
pub(crate) use rustfs_storage_api::{
CompletePart, DeletedObject, HTTPRangeSpec, HealOperations as StorageHealOperations, ListMultipartsInfo,
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
ListOperations as StorageListOperations, ListPartsInfo, MultipartInfo, MultipartOperations as StorageMultipartOperations,
MultipartUploadResult, NamespaceLocking as StorageNamespaceLocking, ObjectIO as StorageObjectIO,
ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations as StorageObjectOperations, ObjectToDelete, PartInfo,
StorageAdminApi, WalkOptions as StorageWalkOptions,
};
crates/iam/src/storage_api.rs
+2 -2
View File
@@ -35,8 +35,8 @@ pub(crate) type IamEcstoreError = EcstoreErrorType;
pub(crate) type IamStorageError = EcstoreStorageError;
pub(crate) type IamStorageResult<T> = EcstoreResultType<T>;
pub(crate) type IamStore = EcstoreStore;
pub(crate) type IamConfigObjectInfo = <IamStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
pub(crate) type IamConfigObjectOptions = <IamStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
pub(crate) type IamConfigObjectInfo = <IamStore as ObjectOperations>::ObjectInfo;
pub(crate) type IamConfigObjectOptions = <IamStore as ObjectOperations>::ObjectOptions;
pub(crate) type IamNotificationSys = EcstoreNotificationSys;
pub(crate) type IamEcstoreNotificationPeerErr = EcstoreNotificationPeerErr;
crates/protocols/src/swift/storage_api.rs
+4 -4
View File
@@ -26,10 +26,10 @@ pub(crate) use rustfs_storage_api::{
ObjectOperations,
};
pub type SwiftGetObjectReader = <SwiftStore as rustfs_storage_api::ObjectIO>::GetObjectReader;
pub type SwiftObjectInfo = <SwiftStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
pub type SwiftObjectOptions = <SwiftStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
pub type SwiftPutObjReader = <SwiftStore as rustfs_storage_api::ObjectIO>::PutObjectReader;
pub type SwiftGetObjectReader = <SwiftStore as ObjectIO>::GetObjectReader;
pub type SwiftObjectInfo = <SwiftStore as ObjectOperations>::ObjectInfo;
pub type SwiftObjectOptions = <SwiftStore as ObjectOperations>::ObjectOptions;
pub type SwiftPutObjReader = <SwiftStore as ObjectIO>::PutObjectReader;
pub(crate) async fn get_swift_bucket_metadata(bucket: &str) -> SwiftStorageResult<Arc<SwiftBucketMetadata>> {
get_swift_bucket_metadata_from_backend(bucket).await
crates/s3select-api/src/storage_api.rs
+3 -3
View File
@@ -25,9 +25,9 @@ pub(crate) use rustfs_ecstore::api::set_disk::DEFAULT_READ_BUFFER_SIZE as SELECT
pub(crate) use rustfs_ecstore::api::storage::ECStore as SelectStore;
pub(crate) use rustfs_storage_api::{HTTPRangeSpec, ObjectIO, ObjectOperations};
pub(crate) type SelectGetObjectReader = <SelectStore as rustfs_storage_api::ObjectIO>::GetObjectReader;
pub(crate) type SelectObjectInfo = <SelectStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
pub(crate) type SelectObjectOptions = <SelectStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
pub(crate) type SelectGetObjectReader = <SelectStore as ObjectIO>::GetObjectReader;
pub(crate) type SelectObjectInfo = <SelectStore as ObjectOperations>::ObjectInfo;
pub(crate) type SelectObjectOptions = <SelectStore as ObjectOperations>::ObjectOptions;
pub(crate) fn resolve_select_object_store_handle() -> Option<Arc<SelectStore>> {
resolve_select_object_store_handle_from_backend()