gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-11 15:46:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: remove ecstore operation compat facades (#3608)

Browse Source
This commit is contained in:
安正超
2026-06-19 17:01:12 +08:00
committed by GitHub
parent 132c8ae77d
commit 54ffbedbc8
42 changed files with 925 additions and 393 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+2 -2
View File
@@ -35,9 +35,9 @@ use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_ver
use crate::event_notification::{EventArgs, send_event};
use crate::global::GLOBAL_LocalNodeName;
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks};
use crate::store::ECStore;
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::tier::warm_backend::WarmBackendGetOpts;
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
use futures::Future;
@@ -2863,9 +2863,9 @@ mod tests {
use crate::disk::endpoint::Endpoint;
use crate::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::error::is_err_invalid_upload_id;
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
use crate::store::ECStore;
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader};
use futures::FutureExt;
use rustfs_common::metrics::{IlmAction, global_metrics};
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
crates/ecstore/src/bucket/lifecycle/core.rs
+1 -1
View File
@@ -26,7 +26,7 @@ use time::{self, Duration, OffsetDateTime};
use tracing::debug;
use uuid::Uuid;
use crate::store_api::ObjectInfo;
use crate::object_api::ObjectInfo;
pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending";
crates/ecstore/src/bucket/lifecycle/tier_delete_journal.rs
+20 -4
View File
@@ -23,9 +23,10 @@ use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_t
use crate::config::com::{delete_config, read_config, save_config};
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::store::ECStore;
use crate::store_api::{ObjectIO, ObjectOperations};
use rustfs_storage_api::ListOperations as _;
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ListOperations as _, ObjectIO, ObjectOperations, ObjectToDelete};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
@@ -104,7 +105,15 @@ fn encode_tier_delete_journal_entry(je: &Jentry) -> Result<Vec<u8>> {
pub async fn persist_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
where
S: ObjectIO,
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?;
save_config(api, &tier_delete_journal_object_name(je), data)
@@ -114,7 +123,14 @@ where
pub async fn remove_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
where
S: ObjectOperations,
S: ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
match delete_config(api, &tier_delete_journal_object_name(je)).await {
Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs
+9 -2
View File
@@ -19,13 +19,20 @@ use tokio_util::sync::CancellationToken;
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::Result;
use crate::object_api::ObjectInfo;
use crate::store::ECStore;
use crate::store_api::{ObjectInfo, ObjectInfoOrErr, WalkOptions};
use rustfs_storage_api::{BucketOperations, BucketOptions, ListOperations as _};
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{
BucketOperations, BucketOptions, ListOperations as _, ObjectInfoOrErr as StorageObjectInfoOrErr,
WalkOptions as StorageWalkOptions,
};
pub const DEFAULT_FREE_VERSION_RECOVERY_LIMIT: usize = 1_000;
const DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT: usize = 10_000;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, crate::error::Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FreeVersionRecoveryStats {
pub scanned: usize,
crates/ecstore/src/bucket/migration.rs
+56 -5
View File
@@ -17,11 +17,18 @@
use crate::bucket::metadata::BUCKET_METADATA_FILE;
use crate::bucket::replication::{decode_resync_file, encode_resync_file};
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::store_api::{ListOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader};
use crate::error::Error;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations};
use http::HeaderMap;
use rustfs_filemeta::FileInfo;
use rustfs_policy::auth::UserIdentity;
use rustfs_policy::policy::PolicyDoc;
use rustfs_storage_api::{BucketOperations, BucketOptions};
use rustfs_storage_api::{
BucketOperations, BucketOptions, DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectIO, ObjectInfoOrErr as StorageObjectInfoOrErr,
ObjectOperations, ObjectToDelete, WalkOptions as StorageWalkOptions,
};
use rustfs_utils::path::SLASH_SEPARATOR;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
@@ -40,6 +47,11 @@ const IAM_POLICY_DB_PREFIX: &str = "config/iam/policydb/";
const REPLICATION_META_DIR: &str = ".replication";
const RESYNC_META_FILE: &str = "resync.bin";
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
#[derive(Debug, Serialize, Deserialize)]
struct CompatIamFormat {
#[serde(default)]
@@ -179,7 +191,23 @@ fn normalize_bucket_meta_blob(path: &str, data: &[u8]) -> std::result::Result<Op
/// Skips buckets that already exist in RustFS (idempotent).
pub async fn try_migrate_bucket_metadata<S>(store: Arc<S>)
where
S: BucketOperations<Error = crate::error::Error> + ObjectIO + ObjectOperations,
S: BucketOperations<Error = crate::error::Error>
+ ObjectIO<
Error = crate::error::Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = crate::error::Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let buckets_list = match store
.list_bucket(&BucketOptions {
@@ -224,7 +252,7 @@ where
async fn migrate_one_if_missing<S>(store: Arc<S>, opts: &ObjectOptions, headers: &HeaderMap, path: &str, label: &str)
where
S: ObjectIO + ObjectOperations,
S: EcstoreObjectIO + EcstoreObjectOperations,
{
if store
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
@@ -278,7 +306,30 @@ where
/// If list_objects_v2 on the legacy bucket fails (e.g. format differs), migration is skipped.
pub async fn try_migrate_iam_config<S>(store: Arc<S>)
where
S: ListOperations + ObjectIO + ObjectOperations,
S: ListOperations<
Error = crate::error::Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = tokio_util::sync::CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + ObjectIO<
Error = crate::error::Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = crate::error::Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let opts = ObjectOptions {
max_parity: true,
crates/ecstore/src/bucket/object_lock/objectlock_sys.rs
+1 -1
View File
@@ -14,7 +14,7 @@
use crate::bucket::metadata_sys::get_object_lock_config;
use crate::bucket::object_lock::objectlock;
use crate::store_api::ObjectInfo;
use crate::object_api::ObjectInfo;
use s3s::dto::{DefaultRetention, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
use std::sync::Arc;
use time::OffsetDateTime;
crates/ecstore/src/bucket/replication/replication_pool.rs
+4 -3
View File
@@ -27,7 +27,8 @@ use crate::bucket::replication::replication_state::ReplicationStats;
use crate::config::com::{read_config, save_config};
use crate::disk::BUCKET_META_PREFIX;
use crate::error::Error as EcstoreError;
use crate::store_api::{ObjectIO, ObjectInfo, ObjectOptions};
use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::storage_api_contracts::EcstoreObjectIO;
use lazy_static::lazy_static;
use rustfs_filemeta::MrfOpKind;
use rustfs_filemeta::MrfReplicateEntry;
@@ -1252,7 +1253,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Returns `true` on success; on failure logs the error and returns `false`.
/// Callers must NOT clear their in-memory buffer on `false` so the next tick
/// can retry — otherwise a transient storage error permanently drops the batch.
async fn flush_mrf_to_disk<S: ObjectIO>(entries: &[MrfReplicateEntry], storage: &Arc<S>) -> bool {
async fn flush_mrf_to_disk<S: EcstoreObjectIO>(entries: &[MrfReplicateEntry], storage: &Arc<S>) -> bool {
match encode_mrf_file(entries) {
Ok(data) => {
if let Err(e) = save_config(storage.clone(), MRF_REPLICATION_FILE, data).await {
@@ -1281,7 +1282,7 @@ async fn flush_mrf_to_disk<S: ObjectIO>(entries: &[MrfReplicateEntry], storage:
}
/// Load bucket resync metadata from disk
async fn load_bucket_resync_metadata<S: ObjectIO>(
async fn load_bucket_resync_metadata<S: EcstoreObjectIO>(
bucket: &str,
obj_api: Arc<S>,
) -> Result<BucketReplicationResyncStatus, EcstoreError> {
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+101 -17
View File
@@ -31,9 +31,10 @@ use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_fo
use crate::event_notification::{EventArgs, send_event};
use crate::global::GLOBAL_LocalNodeName;
use crate::global::get_global_bucket_monitor;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::resolve_object_store_handle;
use crate::set_disk::get_lock_acquire_timeout;
use crate::store_api::{ListOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, WalkOptions};
use crate::storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations};
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput};
use aws_sdk_s3::primitives::ByteStream;
@@ -52,13 +53,17 @@ use http_body_util::StreamBody;
use regex::Regex;
use rmp_serde;
use rustfs_filemeta::{
MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo,
FileInfo, MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo,
ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType,
ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType,
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
};
use rustfs_s3_types::EventName;
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectToDelete};
use rustfs_storage_api::{
DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, NamespaceLocking as StorageNamespaceLocking, ObjectIO,
ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations, ObjectToDelete, WalkOptions as StorageWalkOptions,
};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
@@ -102,10 +107,67 @@ const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed";
const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed";
const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed";
/// Storage capabilities required by bucket replication workers.
pub trait ReplicationStorage: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking {}
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
impl<T> ReplicationStorage for T where T: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking {}
/// Storage capabilities required by bucket replication workers.
pub trait ReplicationStorage:
ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
{
}
impl<T> ReplicationStorage for T where
T: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
{
}
pub(crate) const REPLICATION_DIR: &str = ".replication";
pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
@@ -528,7 +590,18 @@ impl ReplicationResyncer {
}
}
pub async fn mark_status<S: ObjectIO>(&self, status: ResyncStatusType, opts: ResyncOpts, obj_layer: Arc<S>) -> Result<()> {
pub async fn mark_status<S>(&self, status: ResyncStatusType, opts: ResyncOpts, obj_layer: Arc<S>) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let bucket_status = {
let mut status_map = self.status_map.write().await;
let now = OffsetDateTime::now_utc();
@@ -641,7 +714,18 @@ impl ReplicationResyncer {
bucket_status.last_update = Some(now);
}
pub async fn persist_to_disk<S: ObjectIO>(&self, cancel_token: CancellationToken, api: Arc<S>) {
pub async fn persist_to_disk<S>(&self, cancel_token: CancellationToken, api: Arc<S>)
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let mut interval = tokio::time::interval(RESYNC_TIME_INTERVAL);
let mut last_update_times = HashMap::new();
@@ -694,7 +778,7 @@ impl ReplicationResyncer {
}
}
async fn resync_bucket_mark_status<S: ObjectIO>(&self, status: ResyncStatusType, opts: ResyncOpts, storage: Arc<S>) {
async fn resync_bucket_mark_status<S: EcstoreObjectIO>(&self, status: ResyncStatusType, opts: ResyncOpts, storage: Arc<S>) {
if let Err(err) = self.mark_status(status, opts.clone(), storage.clone()).await {
error!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
@@ -1219,7 +1303,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
}
}
pub(crate) async fn save_resync_status<S: ObjectIO>(
pub(crate) async fn save_resync_status<S: EcstoreObjectIO>(
bucket: &str,
status: &BucketReplicationResyncStatus,
api: Arc<S>,
@@ -2181,7 +2265,7 @@ pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicat
}
}
async fn source_delete_marker_missing<S: ObjectOperations>(
async fn source_delete_marker_missing<S: EcstoreObjectOperations>(
storage: &S,
bucket: &str,
object_name: &str,
@@ -2873,13 +2957,13 @@ pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, s
}
trait ReplicateObjectInfoExt {
async fn replicate_object<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
async fn replicate_all<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
async fn replicate_object<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
async fn replicate_all<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
fn to_object_info(&self) -> ObjectInfo;
}
impl ReplicateObjectInfoExt for ReplicateObjectInfo {
async fn replicate_object<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
async fn replicate_object<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
let bucket = self.bucket.clone();
let object = self.name.clone();
@@ -3174,7 +3258,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
rinfo
}
async fn replicate_all<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
async fn replicate_all<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
let start_time = OffsetDateTime::now_utc();
let bucket = self.bucket.clone();
@@ -3896,7 +3980,7 @@ fn part_range_spec_from_actual_size(offset: i64, part_size: i64) -> std::io::Res
))
}
struct MultipartReplicationContext<'a, S: ObjectIO> {
struct MultipartReplicationContext<'a, S: EcstoreObjectIO> {
storage: Arc<S>,
cli: Arc<TargetClient>,
src_bucket: &'a str,
@@ -3908,7 +3992,7 @@ struct MultipartReplicationContext<'a, S: ObjectIO> {
put_opts: PutObjectOptions,
}
async fn replicate_object_with_multipart<S: ObjectIO>(ctx: MultipartReplicationContext<'_, S>) -> std::io::Result<()> {
async fn replicate_object_with_multipart<S: EcstoreObjectIO>(ctx: MultipartReplicationContext<'_, S>) -> std::io::Result<()> {
let MultipartReplicationContext {
storage,
cli,
crates/ecstore/src/client/api_remove.rs
+1 -1
View File
@@ -44,7 +44,7 @@ use crate::client::{
};
use crate::{
disk::DiskAPI,
store_api::{GetObjectReader, ObjectInfo},
object_api::{GetObjectReader, ObjectInfo},
};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
crates/ecstore/src/client/checksum.rs
+1 -1
View File
@@ -26,7 +26,7 @@ use std::collections::HashMap;
use crate::client::utils::base64_decode;
use crate::client::utils::base64_encode;
use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
use crate::{disk::DiskAPI, store_api::GetObjectReader};
use crate::{disk::DiskAPI, object_api::GetObjectReader};
use s3s::header::{
X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
};
crates/ecstore/src/client/object_api_utils.rs
+1 -1
View File
@@ -25,7 +25,7 @@ use std::{collections::HashMap, io::Cursor, sync::Arc};
use tokio::io::BufReader;
use crate::error::ErrorResponse;
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use rustfs_filemeta::ObjectPartInfo;
use rustfs_rio::HashReader;
use rustfs_storage_api::HTTPRangeSpec;
crates/ecstore/src/client/object_handlers_common.rs
+1 -1
View File
@@ -24,8 +24,8 @@ use crate::bucket::lifecycle::lifecycle;
use crate::bucket::replication::{DeletedObjectReplicationInfo, check_replicate_delete, schedule_replication_delete};
use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::object_api::ObjectOptions;
use crate::store::ECStore;
use crate::store_api::ObjectOptions;
use rustfs_filemeta::{REPLICATE_INCOMING_DELETE, ReplicationState, version_purge_statuses_map};
use rustfs_lock::MAX_DELETE_LIST;
use rustfs_storage_api::{ObjectOperations as _, ObjectToDelete};
crates/ecstore/src/client/transition_api.rs
+1 -1
View File
@@ -33,7 +33,7 @@ use crate::client::{
credentials::{CredContext, Credentials, SignatureType, Static},
signer_error,
};
use crate::{client::checksum::ChecksumMode, store_api::GetObjectReader};
use crate::{client::checksum::ChecksumMode, object_api::GetObjectReader};
use futures::{Future, StreamExt};
use http::{HeaderMap, HeaderName};
use http::{
crates/ecstore/src/config/com.rs
+116 -19
View File
@@ -16,7 +16,8 @@ use crate::config::{audit, notify, oidc, set_global_storage_class, storageclass}
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::{Error, Result};
use crate::global::is_first_cluster_node_local;
use crate::store_api::{ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, PutObjReader};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::storage_api_contracts::EcstoreObjectIO;
use http::HeaderMap;
use rustfs_config::audit::{
AUDIT_AMQP_KEYS, AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS,
@@ -32,7 +33,8 @@ use rustfs_config::notify::{
use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
use rustfs_config::server_config::{Config, KVS};
use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
use rustfs_storage_api::StorageAdminApi;
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete, StorageAdminApi};
use rustfs_utils::path::SLASH_SEPARATOR;
use serde_json::{Map, Value};
use std::collections::{HashMap, HashSet};
@@ -182,12 +184,34 @@ fn audit_target_descriptors() -> [TargetConfigDescriptor; 9] {
}
#[instrument(skip(api))]
pub async fn read_config<S: ObjectIO>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
pub async fn read_config<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
Ok(data)
}
pub async fn read_config_no_lock<S: ObjectIO>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let (data, _obj) = read_config_with_metadata(
api,
file,
@@ -200,11 +224,18 @@ pub async fn read_config_no_lock<S: ObjectIO>(api: Arc<S>, file: &str) -> Result
Ok(data)
}
pub async fn read_config_with_metadata<S: ObjectIO>(
api: Arc<S>,
file: &str,
opts: &ObjectOptions,
) -> Result<(Vec<u8>, ObjectInfo)> {
pub async fn read_config_with_metadata<S>(api: Arc<S>, file: &str, opts: &ObjectOptions) -> Result<(Vec<u8>, ObjectInfo)>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let h = HeaderMap::new();
let mut rd = api
.get_object_reader(RUSTFS_META_BUCKET, file, None, h, opts)
@@ -228,7 +259,18 @@ pub async fn read_config_with_metadata<S: ObjectIO>(
}
#[instrument(skip(api, data))]
pub async fn save_config<S: ObjectIO>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
pub async fn save_config<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
save_config_with_opts(
api,
file,
@@ -242,7 +284,17 @@ pub async fn save_config<S: ObjectIO>(api: Arc<S>, file: &str, data: Vec<u8>) ->
}
#[instrument(skip(api))]
pub async fn delete_config<S: ObjectOperations>(api: Arc<S>, file: &str) -> Result<()> {
pub async fn delete_config<S>(api: Arc<S>, file: &str) -> Result<()>
where
S: ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
match api
.delete_object(
RUSTFS_META_BUCKET,
@@ -266,7 +318,18 @@ pub async fn delete_config<S: ObjectOperations>(api: Arc<S>, file: &str) -> Resu
}
}
pub async fn save_config_with_opts<S: ObjectIO>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
pub async fn save_config_with_opts<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let mut put_data = PutObjReader::from_vec(data);
if let Err(err) = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
error!("save_config_with_opts: err: {:?}, file: {}", err, file);
@@ -281,7 +344,7 @@ fn new_server_config() -> Config {
async fn new_and_save_server_config<S>(api: Arc<S>) -> Result<Config>
where
S: ObjectIO + StorageAdminApi,
S: EcstoreObjectIO + StorageAdminApi,
{
let mut cfg = new_server_config();
lookup_configs(&mut cfg, api.clone()).await;
@@ -985,7 +1048,22 @@ fn is_object_not_found(err: &Error) -> bool {
pub async fn try_migrate_server_config<S>(api: Arc<S>)
where
S: ObjectIO + ObjectOperations,
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let config_file = get_config_file();
match api
@@ -1069,7 +1147,7 @@ where
/// Handle the situation where the configuration file does not exist, create and save a new configuration
async fn handle_missing_config<S>(api: Arc<S>, context: &str) -> Result<Config>
where
S: ObjectIO + StorageAdminApi,
S: EcstoreObjectIO + StorageAdminApi,
{
warn!("Configuration not found ({}): Start initializing new configuration", context);
let cfg = if is_first_cluster_node_local().await {
@@ -1091,7 +1169,15 @@ fn handle_config_read_error(err: Error, file_path: &str) -> Result<Config> {
pub async fn read_config_without_migrate<S>(api: Arc<S>) -> Result<Config>
where
S: ObjectIO + StorageAdminApi,
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + StorageAdminApi,
{
let config_file = get_config_file();
@@ -1105,7 +1191,7 @@ where
async fn read_server_config<S>(api: Arc<S>, data: &[u8]) -> Result<Config>
where
S: ObjectIO + StorageAdminApi,
S: EcstoreObjectIO + StorageAdminApi,
{
// If the provided data is empty, try to read from the file again
if data.is_empty() {
@@ -1129,7 +1215,18 @@ where
Ok(cfg.merge())
}
pub async fn save_server_config<S: ObjectIO>(api: Arc<S>, cfg: &Config) -> Result<()> {
pub async fn save_server_config<S>(api: Arc<S>, cfg: &Config) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let config_file = get_config_file();
let existing = match read_config(api.clone(), &config_file).await {
Ok(v) => Some(v),
@@ -1216,8 +1313,8 @@ mod tests {
use crate::endpoints::SetupType;
use crate::error::{Error, Result};
use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::set_disk::SetDisks;
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use http::HeaderMap;
use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{
crates/ecstore/src/data_movement.rs
+1 -1
View File
@@ -13,8 +13,8 @@
// limitations under the License.
use crate::error::{Error, Result, is_err_data_movement_overwrite, is_err_object_not_found, is_err_version_not_found};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::store::ECStore;
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use bytes::Bytes;
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
use rustfs_storage_api::{CompletePart, MultipartOperations as _, ObjectIO as _, ObjectOperations as _};
crates/ecstore/src/data_usage.rs
+1 -1
View File
@@ -715,7 +715,7 @@ pub fn cache_to_data_usage_info(cache: &DataUsageCache, path: &str, buckets: &[r
// Helper functions for DataUsageCache operations
pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str) -> crate::error::Result<DataUsageCache> {
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::store_api::ObjectOptions;
use crate::object_api::ObjectOptions;
use http::HeaderMap;
use rand::RngExt;
use std::path::Path;
crates/ecstore/src/event_notification.rs
+1 -1
View File
@@ -17,8 +17,8 @@
use crate::bucket::metadata::BucketMetadata;
// use crate::event::name::EventName;
use crate::event::targetlist::TargetList;
use crate::object_api::ObjectInfo;
use crate::store::ECStore;
use crate::store_api::ObjectInfo;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::OnceLock;
crates/ecstore/src/lib.rs
+2 -1
View File
@@ -32,14 +32,15 @@ pub mod error;
pub mod global;
pub mod metrics_realtime;
pub mod notification_sys;
pub mod object_api;
pub mod pools;
pub mod rebalance;
pub mod rio;
pub mod rpc;
pub mod set_disk;
mod sets;
mod storage_api_contracts;
pub mod store;
pub mod store_api;
mod store_init;
pub mod store_list_objects;
pub mod store_utils;
crates/ecstore/src/store_api.rs → crates/ecstore/src/object_api/mod.rs
-6
View File
@@ -21,14 +21,11 @@ use crate::store_utils::clean_metadata;
use crate::{bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, bucket::lifecycle::lifecycle::TransitionOptions};
use bytes::Bytes;
use http::{HeaderMap, HeaderValue};
use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{
FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, ReplicateDecision, ReplicationState, ReplicationStatusType,
RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
version_purge_statuses_map,
};
use rustfs_lock::NamespaceLockWrapper;
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_rio::Checksum;
use rustfs_storage_api::{ExpirationOptions, HTTPRangeSpec, TransitionedObject};
use rustfs_utils::CompressionAlgorithm;
@@ -43,7 +40,6 @@ use std::sync::Arc;
use std::task::{Context, Poll};
use time::OffsetDateTime;
use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
use tokio_util::sync::CancellationToken;
use tracing::warn;
use uuid::Uuid;
@@ -51,9 +47,7 @@ pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
mod readers;
mod traits;
mod types;
pub use readers::*;
pub use traits::*;
pub use types::*;
crates/ecstore/src/store_api/readers.rs → crates/ecstore/src/object_api/readers.rs
View File
crates/ecstore/src/store_api/types.rs → crates/ecstore/src/object_api/types.rs
+1 -12
View File
@@ -4,14 +4,6 @@ use rustfs_storage_api::{
VersionMarker,
};
pub type ListObjectsInfo = rustfs_storage_api::ListObjectsInfo<ObjectInfo>;
pub type ListObjectsV2Info = rustfs_storage_api::ListObjectsV2Info<ObjectInfo>;
pub type ListObjectVersionsInfo = rustfs_storage_api::ListObjectVersionsInfo<ObjectInfo>;
pub type ObjectInfoOrErr = rustfs_storage_api::ObjectInfoOrErr<ObjectInfo, Error>;
pub type WalkOptions = rustfs_storage_api::WalkOptions<WalkFilter>;
pub type ObjectToDelete = rustfs_storage_api::ObjectToDelete;
pub type DeletedObject = rustfs_storage_api::DeletedObject;
#[derive(Debug, Default, Clone)]
pub struct ObjectOptions {
// Use the maximum parity (N/2), used when saving server configuration files
@@ -432,7 +424,7 @@ impl ObjectInfo {
restore_expires = restore_status.expiry();
}
// Convert parts from rustfs_filemeta::ObjectPartInfo to store_api::ObjectPartInfo
// Convert parts from rustfs_filemeta::ObjectPartInfo to object_api::ObjectPartInfo
let parts = fi
.parts
.iter()
@@ -715,9 +707,6 @@ fn versions_after_marker(file_infos: &rustfs_filemeta::FileInfoVersions, marker:
.map(|idx| &file_infos.versions[idx + 1..])
.unwrap_or(&file_infos.versions)
}
type WalkFilter = fn(&FileInfo) -> bool;
#[cfg(test)]
mod tests {
use super::*;
crates/ecstore/src/pools.rs
+2 -2
View File
@@ -36,9 +36,9 @@ use crate::error::{
is_err_operation_canceled, is_err_version_not_found,
};
use crate::notification_sys::get_global_notification_sys;
use crate::object_api::{GetObjectReader, ObjectOptions};
use crate::resolve_object_store_handle;
use crate::set_disk::SetDisks;
use crate::store_api::{GetObjectReader, ObjectOptions};
use crate::{global::GLOBAL_LifecycleSys, sets::Sets, store::ECStore};
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered};
@@ -1348,7 +1348,7 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
};
let versioned = BucketVersioningSys::prefix_enabled(bucket, &version.name).await;
let object_info = crate::store_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
let object_info = crate::object_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
let event = eval_action_from_lifecycle(lifecycle_config, lock_retention, replication_config, &object_info).await;
match event.action {
crates/ecstore/src/rebalance.rs
+57 -12
View File
@@ -20,14 +20,17 @@ use crate::disk::error::DiskError;
use crate::error::{Error, Result};
use crate::error::{is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found, is_network_or_host_down};
use crate::global::get_global_endpoints;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::pools::ListCallback;
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
use crate::storage_api_contracts::EcstoreObjectIO;
use crate::store::ECStore;
use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOptions};
use http::HeaderMap;
use rand::RngExt as _;
use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _, StorageAdminApi};
use rustfs_storage_api::{
HTTPRangeSpec, NamespaceLocking as StorageNamespaceLocking, ObjectIO, ObjectOperations as _, StorageAdminApi,
};
use rustfs_utils::path::encode_dir_object;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
@@ -589,11 +592,33 @@ impl RebalanceMeta {
pub fn new() -> Self {
Self::default()
}
pub async fn load<S: ObjectIO>(&mut self, store: Arc<S>) -> Result<()> {
pub async fn load<S>(&mut self, store: Arc<S>) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
self.load_with_opts(store, ObjectOptions::default()).await
}
pub async fn load_with_opts<S: ObjectIO>(&mut self, store: Arc<S>, opts: ObjectOptions) -> Result<()> {
pub async fn load_with_opts<S>(&mut self, store: Arc<S>, opts: ObjectOptions) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let (data, _) = read_config_with_metadata(store, REBAL_META_NAME, &opts).await?;
if data.is_empty() {
debug!(
@@ -634,11 +659,33 @@ impl RebalanceMeta {
Ok(())
}
pub async fn save<S: ObjectIO>(&self, store: Arc<S>) -> Result<()> {
pub async fn save<S>(&self, store: Arc<S>) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
self.save_with_opts(store, ObjectOptions::default()).await
}
pub async fn save_with_opts<S: ObjectIO>(&self, store: Arc<S>, opts: ObjectOptions) -> Result<()> {
pub async fn save_with_opts<S>(&self, store: Arc<S>, opts: ObjectOptions) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
if self.pool_stats.is_empty() {
debug!(
event = EVENT_REBALANCE_STATE,
@@ -667,12 +714,10 @@ impl RebalanceMeta {
}
impl ECStore {
async fn save_rebalance_meta_with_merge<S: ObjectIO + NamespaceLocking>(
&self,
pool: Arc<S>,
local_snapshot: &RebalanceMeta,
stage: &str,
) -> Result<()> {
async fn save_rebalance_meta_with_merge<S>(&self, pool: Arc<S>, local_snapshot: &RebalanceMeta, stage: &str) -> Result<()>
where
S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
{
let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
let _guard = ns_lock
.get_write_lock(get_lock_acquire_timeout())
crates/ecstore/src/set_disk.rs
+12 -6
View File
@@ -33,9 +33,7 @@ use crate::erasure_coding;
use crate::error::{Error, Result, is_err_version_not_found};
use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr};
use crate::store_api::ListObjectVersionsInfo;
use crate::store_api::ObjectOptions;
use crate::store_api::{ObjectInfoOrErr, WalkOptions};
use crate::object_api::ObjectOptions;
use crate::{
bucket::lifecycle::bucket_lifecycle_ops::{
LifecycleOps, gen_transition_objname, get_transitioned_object_reader, put_restore_opts,
@@ -51,7 +49,7 @@ use crate::{
// event::name::EventName,
event_notification::{EventArgs, send_event},
global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure},
store_api::{GetObjectReader, ListObjectsV2Info, MultipartOperations, ObjectIO, ObjectInfo, ObjectOperations, PutObjReader},
object_api::{GetObjectReader, ObjectInfo, PutObjReader},
store_init::load_format_erasure,
};
use bytes::Bytes;
@@ -82,11 +80,14 @@ use rustfs_object_capacity::capacity_scope::{
CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope,
};
use rustfs_s3_types::EventName;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
};
use rustfs_storage_api::{
HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
};
use rustfs_storage_api::{MultipartOperations as _, NamespaceLocking as _, ObjectIO as _, ObjectOperations as _};
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
@@ -131,6 +132,11 @@ use tracing::error;
use tracing::{Instrument, debug, info, warn};
use uuid::Uuid;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_SET_DISK: &str = "set_disk";
const EVENT_SET_DISK_MULTIPART: &str = "set_disk_multipart";
@@ -5014,7 +5020,7 @@ mod tests {
use crate::disk::health_state::RuntimeDriveHealthState;
use crate::endpoints::SetupType;
use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
use crate::store_api::ObjectInfo;
use crate::object_api::ObjectInfo;
use crate::store_init::save_format_file;
use crate::store_list_objects::ListPathOptions;
use rustfs_filemeta::ErasureInfo;
crates/ecstore/src/sets.rs
+10 -4
View File
@@ -15,7 +15,6 @@
use crate::disk::error_reduce::count_errs;
use crate::error::{Error, Result};
use crate::store_api::{ObjectInfoOrErr, WalkOptions};
use crate::{
disk::{
DiskAPI, DiskInfo, DiskOption, DiskStore,
@@ -26,8 +25,8 @@ use crate::{
endpoints::{Endpoints, PoolEndpoints},
error::StorageError,
global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure},
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
set_disk::SetDisks,
store_api::{GetObjectReader, ListObjectVersionsInfo, ListObjectsV2Info, ObjectInfo, ObjectOptions, PutObjReader},
store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
};
use futures::{
@@ -47,8 +46,10 @@ use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_storage_api::CompletePart;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DeletedObject, ListMultipartsInfo, ListPartsInfo,
MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListPartsInfo,
MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectToDelete, PartInfo,
WalkOptions as StorageWalkOptions,
};
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
@@ -61,6 +62,11 @@ use tracing::warn;
use tracing::{error, info};
use uuid::Uuid;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
#[derive(Debug, Clone)]
pub struct Sets {
pub id: Uuid,
crates/ecstore/src/storage_api_contracts.rs
+105
View File
@@ -0,0 +1,105 @@
use crate::error::Error;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{
DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectToDelete,
WalkOptions as StorageWalkOptions,
};
use std::fmt::Debug;
use tokio_util::sync::CancellationToken;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
pub(crate) trait EcstoreObjectIO:
rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
impl<T> EcstoreObjectIO for T where
T: rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
pub(crate) trait EcstoreObjectOperations:
rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
impl<T> EcstoreObjectOperations for T where
T: rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
pub(crate) trait EcstoreListOperations:
rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + Send
+ Sync
+ Debug
{
}
impl<T> EcstoreListOperations for T where
T: rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + Send
+ Sync
+ Debug
{
}
crates/ecstore/src/store.rs
+10 -3
View File
@@ -53,16 +53,15 @@ use crate::notification_sys::get_global_notification_sys;
use crate::pools::PoolMeta;
use crate::rebalance::RebalanceMeta;
use crate::rpc::RemoteClient;
use crate::store_api::{ListObjectVersionsInfo, ObjectInfoOrErr, WalkOptions};
use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
use crate::tier::tier::TierConfigMgr;
use crate::{
bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk},
endpoints::EndpointServerPools,
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
rpc::S3PeerSys,
sets::Sets,
store_api::{GetObjectReader, ListObjectsV2Info, ObjectInfo, ObjectOptions, PutObjReader},
store_init,
};
use futures::future::join_all;
@@ -75,11 +74,14 @@ use rustfs_config::server_config::{Config, get_global_server_config, set_global_
use rustfs_filemeta::FileInfo;
use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
};
use rustfs_storage_api::{
HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
};
use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf};
use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
use std::cmp::Ordering;
@@ -100,6 +102,11 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, instrument, warn};
use uuid::Uuid;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
/// Check if a directory contains any xl.meta files (indicating actual S3 objects)
/// This is used to determine if a bucket is empty for deletion purposes.
async fn has_xlmeta_files(path: &std::path::Path) -> bool {
crates/ecstore/src/store_api/traits.rs
-160
View File
@@ -1,160 +0,0 @@
use super::*;
use rustfs_storage_api::{
CompletePart, DeletedObject, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, ObjectToDelete,
PartInfo,
};
pub trait ObjectIO:
rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
impl<T> ObjectIO for T where
T: rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
/// Object-level storage operations (beyond basic I/O in ObjectIO).
pub trait ObjectOperations:
rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
impl<T> ObjectOperations for T where
T: rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
/// Listing and walking operations.
pub trait ListOperations:
rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + Send
+ Sync
+ Debug
{
}
impl<T> ListOperations for T where
T: rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + Send
+ Sync
+ Debug
{
}
/// Multipart upload operations.
pub trait MultipartOperations:
rustfs_storage_api::MultipartOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
PutObjectReader = PutObjReader,
CompletePart = CompletePart,
ListMultipartsInfo = ListMultipartsInfo,
MultipartUploadResult = MultipartUploadResult,
PartInfo = PartInfo,
MultipartInfo = MultipartInfo,
ListPartsInfo = ListPartsInfo,
> + Send
+ Sync
+ Debug
{
}
impl<T> MultipartOperations for T where
T: rustfs_storage_api::MultipartOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
PutObjectReader = PutObjReader,
CompletePart = CompletePart,
ListMultipartsInfo = ListMultipartsInfo,
MultipartUploadResult = MultipartUploadResult,
PartInfo = PartInfo,
MultipartInfo = MultipartInfo,
ListPartsInfo = ListPartsInfo,
> + Send
+ Sync
+ Debug
{
}
/// Healing and repair operations.
pub trait HealOperations:
rustfs_storage_api::HealOperations<Error = Error, HealResultItem = HealResultItem, HealOptions = HealOpts> + Send + Sync + Debug
{
}
impl<T> HealOperations for T where
T: rustfs_storage_api::HealOperations<Error = Error, HealResultItem = HealResultItem, HealOptions = HealOpts>
+ Send
+ Sync
+ Debug
{
}
/// Namespace lock operations needed by consumers that only coordinate object
/// mutations but do not require the full storage API surface.
pub trait NamespaceLocking:
rustfs_storage_api::NamespaceLocking<Error = Error, NamespaceLock = NamespaceLockWrapper> + Send + Sync + Debug + 'static
{
}
impl<T> NamespaceLocking for T where
T: rustfs_storage_api::NamespaceLocking<Error = Error, NamespaceLock = NamespaceLockWrapper> + Send + Sync + Debug + 'static
{
}
crates/ecstore/src/store_list_objects.rs
+13 -3
View File
@@ -21,17 +21,21 @@ use crate::disk::{DiskInfo, DiskStore};
use crate::error::{
Error, Result, StorageError, is_all_not_found, is_all_volume_not_found, is_err_bucket_not_found, to_object_err,
};
use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::set_disk::SetDisks;
use crate::store_api::{ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectInfoOrErr, ObjectOptions, WalkOptions};
use crate::store::ECStore;
use crate::store_utils::is_reserved_or_invalid_bucket;
use crate::{store::ECStore, store_api::ListObjectsV2Info};
use futures::future::join_all;
use rand::seq::SliceRandom;
use rustfs_filemeta::{
MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
merge_file_meta_versions,
};
use rustfs_storage_api::{ObjectOperations as _, VersionMarker, WalkVersionsSortOrder};
use rustfs_storage_api::{
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo as StorageListObjectsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations as _,
VersionMarker, WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
};
use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
@@ -48,6 +52,12 @@ const MAX_OBJECT_LIST: i32 = 1000;
const METACACHE_SHARE_PREFIX: bool = false;
type ListObjectsInfo = StorageListObjectsInfo<ObjectInfo>;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
fn normalize_max_keys(max_keys: i32) -> i32 {
max_keys.min(MAX_OBJECT_LIST)
}
crates/ecstore/src/tier/tier.rs
+70 -16
View File
@@ -49,11 +49,13 @@ use crate::{
config::com::{CONFIG_PREFIX, read_config},
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
global::is_first_cluster_node_local,
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations},
store::ECStore,
store_api::{ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
};
use rustfs_filemeta::FileInfo;
use rustfs_rio::HashReader;
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::S3ErrorCode;
@@ -1032,19 +1034,36 @@ impl TierConfigMgr {
self.save_tiering_config(api).await
}
pub async fn save_tiering_config<S: ObjectIO>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
pub async fn save_tiering_config<S>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let data = encode_external_tiering_config_blob(self)?;
let config_file = tier_config_path(TIER_CONFIG_FILE);
self.save_config(api, &config_file, data).await
}
pub async fn save_config<S: ObjectIO>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
) -> std::result::Result<(), std::io::Error> {
pub async fn save_config<S>(&self, api: Arc<S>, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
self.save_config_with_opts(
api,
file,
@@ -1057,13 +1076,24 @@ impl TierConfigMgr {
.await
}
pub async fn save_config_with_opts<S: ObjectIO>(
pub async fn save_config_with_opts<S>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
opts: &ObjectOptions,
) -> std::result::Result<(), std::io::Error> {
) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
debug!("save tier config:{}", file);
let mut put_data = PutObjReader::from_vec(data.to_vec());
let _ = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await?;
@@ -1099,7 +1129,10 @@ impl TierConfigMgr {
}
}
async fn new_and_save_tiering_config<S: ObjectIO>(api: Arc<S>) -> Result<TierConfigMgr> {
async fn new_and_save_tiering_config<S>(api: Arc<S>) -> Result<TierConfigMgr>
where
S: EcstoreObjectIO,
{
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
@@ -1158,12 +1191,15 @@ async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMg
}
}
async fn read_tier_config_from_bucket<S: ObjectIO>(
async fn read_tier_config_from_bucket<S>(
api: Arc<S>,
bucket: &str,
path: &str,
opts: &ObjectOptions,
) -> io::Result<Option<Vec<u8>>> {
) -> io::Result<Option<Vec<u8>>>
where
S: EcstoreObjectIO,
{
let mut rd = match api.get_object_reader(bucket, path, None, HeaderMap::new(), opts).await {
Ok(v) => v,
Err(err) if is_err_config_not_found(&err) => return Ok(None),
@@ -1176,7 +1212,10 @@ async fn read_tier_config_from_bucket<S: ObjectIO>(
Ok(Some(data))
}
async fn write_tier_config_to_rustfs<S: ObjectIO>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()> {
async fn write_tier_config_to_rustfs<S>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()>
where
S: EcstoreObjectIO,
{
let mut put_data = PutObjReader::from_vec(data.to_vec());
api.put_object(
RUSTFS_META_BUCKET,
@@ -1194,7 +1233,22 @@ async fn write_tier_config_to_rustfs<S: ObjectIO>(api: Arc<S>, path: &str, data:
pub async fn try_migrate_tiering_config<S>(api: Arc<S>)
where
S: ObjectIO + ObjectOperations,
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let target_path = tier_config_path(TIER_CONFIG_FILE);
if api