refactor: remove ecstore operation compat facades (#3608)

This commit is contained in:
安正超
2026-06-19 17:01:12 +08:00
committed by GitHub
parent 132c8ae77d
commit 54ffbedbc8
42 changed files with 925 additions and 393 deletions
@@ -35,9 +35,9 @@ use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_ver
use crate::event_notification::{EventArgs, send_event};
use crate::global::GLOBAL_LocalNodeName;
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks};
use crate::store::ECStore;
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::tier::warm_backend::WarmBackendGetOpts;
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
use futures::Future;
@@ -2863,9 +2863,9 @@ mod tests {
use crate::disk::endpoint::Endpoint;
use crate::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::error::is_err_invalid_upload_id;
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
use crate::store::ECStore;
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader};
use futures::FutureExt;
use rustfs_common::metrics::{IlmAction, global_metrics};
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
+1 -1
View File
@@ -26,7 +26,7 @@ use time::{self, Duration, OffsetDateTime};
use tracing::debug;
use uuid::Uuid;
use crate::store_api::ObjectInfo;
use crate::object_api::ObjectInfo;
pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending";
@@ -23,9 +23,10 @@ use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_t
use crate::config::com::{delete_config, read_config, save_config};
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::store::ECStore;
use crate::store_api::{ObjectIO, ObjectOperations};
use rustfs_storage_api::ListOperations as _;
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ListOperations as _, ObjectIO, ObjectOperations, ObjectToDelete};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
@@ -104,7 +105,15 @@ fn encode_tier_delete_journal_entry(je: &Jentry) -> Result<Vec<u8>> {
pub async fn persist_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
where
S: ObjectIO,
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?;
save_config(api, &tier_delete_journal_object_name(je), data)
@@ -114,7 +123,14 @@ where
pub async fn remove_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
where
S: ObjectOperations,
S: ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
match delete_config(api, &tier_delete_journal_object_name(je)).await {
Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
@@ -19,13 +19,20 @@ use tokio_util::sync::CancellationToken;
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::Result;
use crate::object_api::ObjectInfo;
use crate::store::ECStore;
use crate::store_api::{ObjectInfo, ObjectInfoOrErr, WalkOptions};
use rustfs_storage_api::{BucketOperations, BucketOptions, ListOperations as _};
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{
BucketOperations, BucketOptions, ListOperations as _, ObjectInfoOrErr as StorageObjectInfoOrErr,
WalkOptions as StorageWalkOptions,
};
pub const DEFAULT_FREE_VERSION_RECOVERY_LIMIT: usize = 1_000;
const DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT: usize = 10_000;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, crate::error::Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FreeVersionRecoveryStats {
pub scanned: usize,
+56 -5
View File
@@ -17,11 +17,18 @@
use crate::bucket::metadata::BUCKET_METADATA_FILE;
use crate::bucket::replication::{decode_resync_file, encode_resync_file};
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::store_api::{ListOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader};
use crate::error::Error;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations};
use http::HeaderMap;
use rustfs_filemeta::FileInfo;
use rustfs_policy::auth::UserIdentity;
use rustfs_policy::policy::PolicyDoc;
use rustfs_storage_api::{BucketOperations, BucketOptions};
use rustfs_storage_api::{
BucketOperations, BucketOptions, DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectIO, ObjectInfoOrErr as StorageObjectInfoOrErr,
ObjectOperations, ObjectToDelete, WalkOptions as StorageWalkOptions,
};
use rustfs_utils::path::SLASH_SEPARATOR;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
@@ -40,6 +47,11 @@ const IAM_POLICY_DB_PREFIX: &str = "config/iam/policydb/";
const REPLICATION_META_DIR: &str = ".replication";
const RESYNC_META_FILE: &str = "resync.bin";
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
#[derive(Debug, Serialize, Deserialize)]
struct CompatIamFormat {
#[serde(default)]
@@ -179,7 +191,23 @@ fn normalize_bucket_meta_blob(path: &str, data: &[u8]) -> std::result::Result<Op
/// Skips buckets that already exist in RustFS (idempotent).
pub async fn try_migrate_bucket_metadata<S>(store: Arc<S>)
where
S: BucketOperations<Error = crate::error::Error> + ObjectIO + ObjectOperations,
S: BucketOperations<Error = crate::error::Error>
+ ObjectIO<
Error = crate::error::Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = crate::error::Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let buckets_list = match store
.list_bucket(&BucketOptions {
@@ -224,7 +252,7 @@ where
async fn migrate_one_if_missing<S>(store: Arc<S>, opts: &ObjectOptions, headers: &HeaderMap, path: &str, label: &str)
where
S: ObjectIO + ObjectOperations,
S: EcstoreObjectIO + EcstoreObjectOperations,
{
if store
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
@@ -278,7 +306,30 @@ where
/// If list_objects_v2 on the legacy bucket fails (e.g. format differs), migration is skipped.
pub async fn try_migrate_iam_config<S>(store: Arc<S>)
where
S: ListOperations + ObjectIO + ObjectOperations,
S: ListOperations<
Error = crate::error::Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = tokio_util::sync::CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + ObjectIO<
Error = crate::error::Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = crate::error::Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let opts = ObjectOptions {
max_parity: true,
@@ -14,7 +14,7 @@
use crate::bucket::metadata_sys::get_object_lock_config;
use crate::bucket::object_lock::objectlock;
use crate::store_api::ObjectInfo;
use crate::object_api::ObjectInfo;
use s3s::dto::{DefaultRetention, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
use std::sync::Arc;
use time::OffsetDateTime;
@@ -27,7 +27,8 @@ use crate::bucket::replication::replication_state::ReplicationStats;
use crate::config::com::{read_config, save_config};
use crate::disk::BUCKET_META_PREFIX;
use crate::error::Error as EcstoreError;
use crate::store_api::{ObjectIO, ObjectInfo, ObjectOptions};
use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::storage_api_contracts::EcstoreObjectIO;
use lazy_static::lazy_static;
use rustfs_filemeta::MrfOpKind;
use rustfs_filemeta::MrfReplicateEntry;
@@ -1252,7 +1253,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Returns `true` on success; on failure logs the error and returns `false`.
/// Callers must NOT clear their in-memory buffer on `false` so the next tick
/// can retry — otherwise a transient storage error permanently drops the batch.
async fn flush_mrf_to_disk<S: ObjectIO>(entries: &[MrfReplicateEntry], storage: &Arc<S>) -> bool {
async fn flush_mrf_to_disk<S: EcstoreObjectIO>(entries: &[MrfReplicateEntry], storage: &Arc<S>) -> bool {
match encode_mrf_file(entries) {
Ok(data) => {
if let Err(e) = save_config(storage.clone(), MRF_REPLICATION_FILE, data).await {
@@ -1281,7 +1282,7 @@ async fn flush_mrf_to_disk<S: ObjectIO>(entries: &[MrfReplicateEntry], storage:
}
/// Load bucket resync metadata from disk
async fn load_bucket_resync_metadata<S: ObjectIO>(
async fn load_bucket_resync_metadata<S: EcstoreObjectIO>(
bucket: &str,
obj_api: Arc<S>,
) -> Result<BucketReplicationResyncStatus, EcstoreError> {
@@ -31,9 +31,10 @@ use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_fo
use crate::event_notification::{EventArgs, send_event};
use crate::global::GLOBAL_LocalNodeName;
use crate::global::get_global_bucket_monitor;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::resolve_object_store_handle;
use crate::set_disk::get_lock_acquire_timeout;
use crate::store_api::{ListOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, WalkOptions};
use crate::storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations};
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput};
use aws_sdk_s3::primitives::ByteStream;
@@ -52,13 +53,17 @@ use http_body_util::StreamBody;
use regex::Regex;
use rmp_serde;
use rustfs_filemeta::{
MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo,
FileInfo, MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo,
ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType,
ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType,
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
};
use rustfs_s3_types::EventName;
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectToDelete};
use rustfs_storage_api::{
DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, NamespaceLocking as StorageNamespaceLocking, ObjectIO,
ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations, ObjectToDelete, WalkOptions as StorageWalkOptions,
};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
@@ -102,10 +107,67 @@ const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed";
const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed";
const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed";
/// Storage capabilities required by bucket replication workers.
pub trait ReplicationStorage: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking {}
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
impl<T> ReplicationStorage for T where T: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking {}
/// Storage capabilities required by bucket replication workers.
pub trait ReplicationStorage:
ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
{
}
impl<T> ReplicationStorage for T where
T: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
{
}
pub(crate) const REPLICATION_DIR: &str = ".replication";
pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
@@ -528,7 +590,18 @@ impl ReplicationResyncer {
}
}
pub async fn mark_status<S: ObjectIO>(&self, status: ResyncStatusType, opts: ResyncOpts, obj_layer: Arc<S>) -> Result<()> {
pub async fn mark_status<S>(&self, status: ResyncStatusType, opts: ResyncOpts, obj_layer: Arc<S>) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let bucket_status = {
let mut status_map = self.status_map.write().await;
let now = OffsetDateTime::now_utc();
@@ -641,7 +714,18 @@ impl ReplicationResyncer {
bucket_status.last_update = Some(now);
}
pub async fn persist_to_disk<S: ObjectIO>(&self, cancel_token: CancellationToken, api: Arc<S>) {
pub async fn persist_to_disk<S>(&self, cancel_token: CancellationToken, api: Arc<S>)
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let mut interval = tokio::time::interval(RESYNC_TIME_INTERVAL);
let mut last_update_times = HashMap::new();
@@ -694,7 +778,7 @@ impl ReplicationResyncer {
}
}
async fn resync_bucket_mark_status<S: ObjectIO>(&self, status: ResyncStatusType, opts: ResyncOpts, storage: Arc<S>) {
async fn resync_bucket_mark_status<S: EcstoreObjectIO>(&self, status: ResyncStatusType, opts: ResyncOpts, storage: Arc<S>) {
if let Err(err) = self.mark_status(status, opts.clone(), storage.clone()).await {
error!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
@@ -1219,7 +1303,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
}
}
pub(crate) async fn save_resync_status<S: ObjectIO>(
pub(crate) async fn save_resync_status<S: EcstoreObjectIO>(
bucket: &str,
status: &BucketReplicationResyncStatus,
api: Arc<S>,
@@ -2181,7 +2265,7 @@ pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicat
}
}
async fn source_delete_marker_missing<S: ObjectOperations>(
async fn source_delete_marker_missing<S: EcstoreObjectOperations>(
storage: &S,
bucket: &str,
object_name: &str,
@@ -2873,13 +2957,13 @@ pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, s
}
trait ReplicateObjectInfoExt {
async fn replicate_object<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
async fn replicate_all<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
async fn replicate_object<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
async fn replicate_all<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
fn to_object_info(&self) -> ObjectInfo;
}
impl ReplicateObjectInfoExt for ReplicateObjectInfo {
async fn replicate_object<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
async fn replicate_object<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
let bucket = self.bucket.clone();
let object = self.name.clone();
@@ -3174,7 +3258,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
rinfo
}
async fn replicate_all<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
async fn replicate_all<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
let start_time = OffsetDateTime::now_utc();
let bucket = self.bucket.clone();
@@ -3896,7 +3980,7 @@ fn part_range_spec_from_actual_size(offset: i64, part_size: i64) -> std::io::Res
))
}
struct MultipartReplicationContext<'a, S: ObjectIO> {
struct MultipartReplicationContext<'a, S: EcstoreObjectIO> {
storage: Arc<S>,
cli: Arc<TargetClient>,
src_bucket: &'a str,
@@ -3908,7 +3992,7 @@ struct MultipartReplicationContext<'a, S: ObjectIO> {
put_opts: PutObjectOptions,
}
async fn replicate_object_with_multipart<S: ObjectIO>(ctx: MultipartReplicationContext<'_, S>) -> std::io::Result<()> {
async fn replicate_object_with_multipart<S: EcstoreObjectIO>(ctx: MultipartReplicationContext<'_, S>) -> std::io::Result<()> {
let MultipartReplicationContext {
storage,
cli,
+1 -1
View File
@@ -44,7 +44,7 @@ use crate::client::{
};
use crate::{
disk::DiskAPI,
store_api::{GetObjectReader, ObjectInfo},
object_api::{GetObjectReader, ObjectInfo},
};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+1 -1
View File
@@ -26,7 +26,7 @@ use std::collections::HashMap;
use crate::client::utils::base64_decode;
use crate::client::utils::base64_encode;
use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
use crate::{disk::DiskAPI, store_api::GetObjectReader};
use crate::{disk::DiskAPI, object_api::GetObjectReader};
use s3s::header::{
X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
};
@@ -25,7 +25,7 @@ use std::{collections::HashMap, io::Cursor, sync::Arc};
use tokio::io::BufReader;
use crate::error::ErrorResponse;
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use rustfs_filemeta::ObjectPartInfo;
use rustfs_rio::HashReader;
use rustfs_storage_api::HTTPRangeSpec;
@@ -24,8 +24,8 @@ use crate::bucket::lifecycle::lifecycle;
use crate::bucket::replication::{DeletedObjectReplicationInfo, check_replicate_delete, schedule_replication_delete};
use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::object_api::ObjectOptions;
use crate::store::ECStore;
use crate::store_api::ObjectOptions;
use rustfs_filemeta::{REPLICATE_INCOMING_DELETE, ReplicationState, version_purge_statuses_map};
use rustfs_lock::MAX_DELETE_LIST;
use rustfs_storage_api::{ObjectOperations as _, ObjectToDelete};
+1 -1
View File
@@ -33,7 +33,7 @@ use crate::client::{
credentials::{CredContext, Credentials, SignatureType, Static},
signer_error,
};
use crate::{client::checksum::ChecksumMode, store_api::GetObjectReader};
use crate::{client::checksum::ChecksumMode, object_api::GetObjectReader};
use futures::{Future, StreamExt};
use http::{HeaderMap, HeaderName};
use http::{
+116 -19
View File
@@ -16,7 +16,8 @@ use crate::config::{audit, notify, oidc, set_global_storage_class, storageclass}
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::{Error, Result};
use crate::global::is_first_cluster_node_local;
use crate::store_api::{ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, PutObjReader};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::storage_api_contracts::EcstoreObjectIO;
use http::HeaderMap;
use rustfs_config::audit::{
AUDIT_AMQP_KEYS, AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS,
@@ -32,7 +33,8 @@ use rustfs_config::notify::{
use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
use rustfs_config::server_config::{Config, KVS};
use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
use rustfs_storage_api::StorageAdminApi;
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete, StorageAdminApi};
use rustfs_utils::path::SLASH_SEPARATOR;
use serde_json::{Map, Value};
use std::collections::{HashMap, HashSet};
@@ -182,12 +184,34 @@ fn audit_target_descriptors() -> [TargetConfigDescriptor; 9] {
}
#[instrument(skip(api))]
pub async fn read_config<S: ObjectIO>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
pub async fn read_config<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
Ok(data)
}
pub async fn read_config_no_lock<S: ObjectIO>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let (data, _obj) = read_config_with_metadata(
api,
file,
@@ -200,11 +224,18 @@ pub async fn read_config_no_lock<S: ObjectIO>(api: Arc<S>, file: &str) -> Result
Ok(data)
}
pub async fn read_config_with_metadata<S: ObjectIO>(
api: Arc<S>,
file: &str,
opts: &ObjectOptions,
) -> Result<(Vec<u8>, ObjectInfo)> {
pub async fn read_config_with_metadata<S>(api: Arc<S>, file: &str, opts: &ObjectOptions) -> Result<(Vec<u8>, ObjectInfo)>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let h = HeaderMap::new();
let mut rd = api
.get_object_reader(RUSTFS_META_BUCKET, file, None, h, opts)
@@ -228,7 +259,18 @@ pub async fn read_config_with_metadata<S: ObjectIO>(
}
#[instrument(skip(api, data))]
pub async fn save_config<S: ObjectIO>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
pub async fn save_config<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
save_config_with_opts(
api,
file,
@@ -242,7 +284,17 @@ pub async fn save_config<S: ObjectIO>(api: Arc<S>, file: &str, data: Vec<u8>) ->
}
#[instrument(skip(api))]
pub async fn delete_config<S: ObjectOperations>(api: Arc<S>, file: &str) -> Result<()> {
pub async fn delete_config<S>(api: Arc<S>, file: &str) -> Result<()>
where
S: ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
match api
.delete_object(
RUSTFS_META_BUCKET,
@@ -266,7 +318,18 @@ pub async fn delete_config<S: ObjectOperations>(api: Arc<S>, file: &str) -> Resu
}
}
pub async fn save_config_with_opts<S: ObjectIO>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
pub async fn save_config_with_opts<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let mut put_data = PutObjReader::from_vec(data);
if let Err(err) = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
error!("save_config_with_opts: err: {:?}, file: {}", err, file);
@@ -281,7 +344,7 @@ fn new_server_config() -> Config {
async fn new_and_save_server_config<S>(api: Arc<S>) -> Result<Config>
where
S: ObjectIO + StorageAdminApi,
S: EcstoreObjectIO + StorageAdminApi,
{
let mut cfg = new_server_config();
lookup_configs(&mut cfg, api.clone()).await;
@@ -985,7 +1048,22 @@ fn is_object_not_found(err: &Error) -> bool {
pub async fn try_migrate_server_config<S>(api: Arc<S>)
where
S: ObjectIO + ObjectOperations,
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let config_file = get_config_file();
match api
@@ -1069,7 +1147,7 @@ where
/// Handle the situation where the configuration file does not exist, create and save a new configuration
async fn handle_missing_config<S>(api: Arc<S>, context: &str) -> Result<Config>
where
S: ObjectIO + StorageAdminApi,
S: EcstoreObjectIO + StorageAdminApi,
{
warn!("Configuration not found ({}): Start initializing new configuration", context);
let cfg = if is_first_cluster_node_local().await {
@@ -1091,7 +1169,15 @@ fn handle_config_read_error(err: Error, file_path: &str) -> Result<Config> {
pub async fn read_config_without_migrate<S>(api: Arc<S>) -> Result<Config>
where
S: ObjectIO + StorageAdminApi,
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + StorageAdminApi,
{
let config_file = get_config_file();
@@ -1105,7 +1191,7 @@ where
async fn read_server_config<S>(api: Arc<S>, data: &[u8]) -> Result<Config>
where
S: ObjectIO + StorageAdminApi,
S: EcstoreObjectIO + StorageAdminApi,
{
// If the provided data is empty, try to read from the file again
if data.is_empty() {
@@ -1129,7 +1215,18 @@ where
Ok(cfg.merge())
}
pub async fn save_server_config<S: ObjectIO>(api: Arc<S>, cfg: &Config) -> Result<()> {
pub async fn save_server_config<S>(api: Arc<S>, cfg: &Config) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let config_file = get_config_file();
let existing = match read_config(api.clone(), &config_file).await {
Ok(v) => Some(v),
@@ -1216,8 +1313,8 @@ mod tests {
use crate::endpoints::SetupType;
use crate::error::{Error, Result};
use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::set_disk::SetDisks;
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use http::HeaderMap;
use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{
+1 -1
View File
@@ -13,8 +13,8 @@
// limitations under the License.
use crate::error::{Error, Result, is_err_data_movement_overwrite, is_err_object_not_found, is_err_version_not_found};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::store::ECStore;
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use bytes::Bytes;
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
use rustfs_storage_api::{CompletePart, MultipartOperations as _, ObjectIO as _, ObjectOperations as _};
+1 -1
View File
@@ -715,7 +715,7 @@ pub fn cache_to_data_usage_info(cache: &DataUsageCache, path: &str, buckets: &[r
// Helper functions for DataUsageCache operations
pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str) -> crate::error::Result<DataUsageCache> {
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::store_api::ObjectOptions;
use crate::object_api::ObjectOptions;
use http::HeaderMap;
use rand::RngExt;
use std::path::Path;
+1 -1
View File
@@ -17,8 +17,8 @@
use crate::bucket::metadata::BucketMetadata;
// use crate::event::name::EventName;
use crate::event::targetlist::TargetList;
use crate::object_api::ObjectInfo;
use crate::store::ECStore;
use crate::store_api::ObjectInfo;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::OnceLock;
+2 -1
View File
@@ -32,14 +32,15 @@ pub mod error;
pub mod global;
pub mod metrics_realtime;
pub mod notification_sys;
pub mod object_api;
pub mod pools;
pub mod rebalance;
pub mod rio;
pub mod rpc;
pub mod set_disk;
mod sets;
mod storage_api_contracts;
pub mod store;
pub mod store_api;
mod store_init;
pub mod store_list_objects;
pub mod store_utils;
@@ -21,14 +21,11 @@ use crate::store_utils::clean_metadata;
use crate::{bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, bucket::lifecycle::lifecycle::TransitionOptions};
use bytes::Bytes;
use http::{HeaderMap, HeaderValue};
use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{
FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, ReplicateDecision, ReplicationState, ReplicationStatusType,
RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
version_purge_statuses_map,
};
use rustfs_lock::NamespaceLockWrapper;
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_rio::Checksum;
use rustfs_storage_api::{ExpirationOptions, HTTPRangeSpec, TransitionedObject};
use rustfs_utils::CompressionAlgorithm;
@@ -43,7 +40,6 @@ use std::sync::Arc;
use std::task::{Context, Poll};
use time::OffsetDateTime;
use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
use tokio_util::sync::CancellationToken;
use tracing::warn;
use uuid::Uuid;
@@ -51,9 +47,7 @@ pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
mod readers;
mod traits;
mod types;
pub use readers::*;
pub use traits::*;
pub use types::*;
@@ -4,14 +4,6 @@ use rustfs_storage_api::{
VersionMarker,
};
pub type ListObjectsInfo = rustfs_storage_api::ListObjectsInfo<ObjectInfo>;
pub type ListObjectsV2Info = rustfs_storage_api::ListObjectsV2Info<ObjectInfo>;
pub type ListObjectVersionsInfo = rustfs_storage_api::ListObjectVersionsInfo<ObjectInfo>;
pub type ObjectInfoOrErr = rustfs_storage_api::ObjectInfoOrErr<ObjectInfo, Error>;
pub type WalkOptions = rustfs_storage_api::WalkOptions<WalkFilter>;
pub type ObjectToDelete = rustfs_storage_api::ObjectToDelete;
pub type DeletedObject = rustfs_storage_api::DeletedObject;
#[derive(Debug, Default, Clone)]
pub struct ObjectOptions {
// Use the maximum parity (N/2), used when saving server configuration files
@@ -432,7 +424,7 @@ impl ObjectInfo {
restore_expires = restore_status.expiry();
}
// Convert parts from rustfs_filemeta::ObjectPartInfo to store_api::ObjectPartInfo
// Convert parts from rustfs_filemeta::ObjectPartInfo to object_api::ObjectPartInfo
let parts = fi
.parts
.iter()
@@ -715,9 +707,6 @@ fn versions_after_marker(file_infos: &rustfs_filemeta::FileInfoVersions, marker:
.map(|idx| &file_infos.versions[idx + 1..])
.unwrap_or(&file_infos.versions)
}
type WalkFilter = fn(&FileInfo) -> bool;
#[cfg(test)]
mod tests {
use super::*;
+2 -2
View File
@@ -36,9 +36,9 @@ use crate::error::{
is_err_operation_canceled, is_err_version_not_found,
};
use crate::notification_sys::get_global_notification_sys;
use crate::object_api::{GetObjectReader, ObjectOptions};
use crate::resolve_object_store_handle;
use crate::set_disk::SetDisks;
use crate::store_api::{GetObjectReader, ObjectOptions};
use crate::{global::GLOBAL_LifecycleSys, sets::Sets, store::ECStore};
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered};
@@ -1348,7 +1348,7 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
};
let versioned = BucketVersioningSys::prefix_enabled(bucket, &version.name).await;
let object_info = crate::store_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
let object_info = crate::object_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
let event = eval_action_from_lifecycle(lifecycle_config, lock_retention, replication_config, &object_info).await;
match event.action {
+57 -12
View File
@@ -20,14 +20,17 @@ use crate::disk::error::DiskError;
use crate::error::{Error, Result};
use crate::error::{is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found, is_network_or_host_down};
use crate::global::get_global_endpoints;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::pools::ListCallback;
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
use crate::storage_api_contracts::EcstoreObjectIO;
use crate::store::ECStore;
use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOptions};
use http::HeaderMap;
use rand::RngExt as _;
use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _, StorageAdminApi};
use rustfs_storage_api::{
HTTPRangeSpec, NamespaceLocking as StorageNamespaceLocking, ObjectIO, ObjectOperations as _, StorageAdminApi,
};
use rustfs_utils::path::encode_dir_object;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
@@ -589,11 +592,33 @@ impl RebalanceMeta {
pub fn new() -> Self {
Self::default()
}
pub async fn load<S: ObjectIO>(&mut self, store: Arc<S>) -> Result<()> {
pub async fn load<S>(&mut self, store: Arc<S>) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
self.load_with_opts(store, ObjectOptions::default()).await
}
pub async fn load_with_opts<S: ObjectIO>(&mut self, store: Arc<S>, opts: ObjectOptions) -> Result<()> {
pub async fn load_with_opts<S>(&mut self, store: Arc<S>, opts: ObjectOptions) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let (data, _) = read_config_with_metadata(store, REBAL_META_NAME, &opts).await?;
if data.is_empty() {
debug!(
@@ -634,11 +659,33 @@ impl RebalanceMeta {
Ok(())
}
pub async fn save<S: ObjectIO>(&self, store: Arc<S>) -> Result<()> {
pub async fn save<S>(&self, store: Arc<S>) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
self.save_with_opts(store, ObjectOptions::default()).await
}
pub async fn save_with_opts<S: ObjectIO>(&self, store: Arc<S>, opts: ObjectOptions) -> Result<()> {
pub async fn save_with_opts<S>(&self, store: Arc<S>, opts: ObjectOptions) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
if self.pool_stats.is_empty() {
debug!(
event = EVENT_REBALANCE_STATE,
@@ -667,12 +714,10 @@ impl RebalanceMeta {
}
impl ECStore {
async fn save_rebalance_meta_with_merge<S: ObjectIO + NamespaceLocking>(
&self,
pool: Arc<S>,
local_snapshot: &RebalanceMeta,
stage: &str,
) -> Result<()> {
async fn save_rebalance_meta_with_merge<S>(&self, pool: Arc<S>, local_snapshot: &RebalanceMeta, stage: &str) -> Result<()>
where
S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
{
let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
let _guard = ns_lock
.get_write_lock(get_lock_acquire_timeout())
+12 -6
View File
@@ -33,9 +33,7 @@ use crate::erasure_coding;
use crate::error::{Error, Result, is_err_version_not_found};
use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr};
use crate::store_api::ListObjectVersionsInfo;
use crate::store_api::ObjectOptions;
use crate::store_api::{ObjectInfoOrErr, WalkOptions};
use crate::object_api::ObjectOptions;
use crate::{
bucket::lifecycle::bucket_lifecycle_ops::{
LifecycleOps, gen_transition_objname, get_transitioned_object_reader, put_restore_opts,
@@ -51,7 +49,7 @@ use crate::{
// event::name::EventName,
event_notification::{EventArgs, send_event},
global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure},
store_api::{GetObjectReader, ListObjectsV2Info, MultipartOperations, ObjectIO, ObjectInfo, ObjectOperations, PutObjReader},
object_api::{GetObjectReader, ObjectInfo, PutObjReader},
store_init::load_format_erasure,
};
use bytes::Bytes;
@@ -82,11 +80,14 @@ use rustfs_object_capacity::capacity_scope::{
CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope,
};
use rustfs_s3_types::EventName;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
};
use rustfs_storage_api::{
HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
};
use rustfs_storage_api::{MultipartOperations as _, NamespaceLocking as _, ObjectIO as _, ObjectOperations as _};
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
@@ -131,6 +132,11 @@ use tracing::error;
use tracing::{Instrument, debug, info, warn};
use uuid::Uuid;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_SET_DISK: &str = "set_disk";
const EVENT_SET_DISK_MULTIPART: &str = "set_disk_multipart";
@@ -5014,7 +5020,7 @@ mod tests {
use crate::disk::health_state::RuntimeDriveHealthState;
use crate::endpoints::SetupType;
use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
use crate::store_api::ObjectInfo;
use crate::object_api::ObjectInfo;
use crate::store_init::save_format_file;
use crate::store_list_objects::ListPathOptions;
use rustfs_filemeta::ErasureInfo;
+10 -4
View File
@@ -15,7 +15,6 @@
use crate::disk::error_reduce::count_errs;
use crate::error::{Error, Result};
use crate::store_api::{ObjectInfoOrErr, WalkOptions};
use crate::{
disk::{
DiskAPI, DiskInfo, DiskOption, DiskStore,
@@ -26,8 +25,8 @@ use crate::{
endpoints::{Endpoints, PoolEndpoints},
error::StorageError,
global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure},
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
set_disk::SetDisks,
store_api::{GetObjectReader, ListObjectVersionsInfo, ListObjectsV2Info, ObjectInfo, ObjectOptions, PutObjReader},
store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
};
use futures::{
@@ -47,8 +46,10 @@ use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_storage_api::CompletePart;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DeletedObject, ListMultipartsInfo, ListPartsInfo,
MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListPartsInfo,
MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectToDelete, PartInfo,
WalkOptions as StorageWalkOptions,
};
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
@@ -61,6 +62,11 @@ use tracing::warn;
use tracing::{error, info};
use uuid::Uuid;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
#[derive(Debug, Clone)]
pub struct Sets {
pub id: Uuid,
+105
View File
@@ -0,0 +1,105 @@
use crate::error::Error;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{
DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectToDelete,
WalkOptions as StorageWalkOptions,
};
use std::fmt::Debug;
use tokio_util::sync::CancellationToken;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
pub(crate) trait EcstoreObjectIO:
rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
impl<T> EcstoreObjectIO for T where
T: rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
pub(crate) trait EcstoreObjectOperations:
rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
impl<T> EcstoreObjectOperations for T where
T: rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
pub(crate) trait EcstoreListOperations:
rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + Send
+ Sync
+ Debug
{
}
impl<T> EcstoreListOperations for T where
T: rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + Send
+ Sync
+ Debug
{
}
+10 -3
View File
@@ -53,16 +53,15 @@ use crate::notification_sys::get_global_notification_sys;
use crate::pools::PoolMeta;
use crate::rebalance::RebalanceMeta;
use crate::rpc::RemoteClient;
use crate::store_api::{ListObjectVersionsInfo, ObjectInfoOrErr, WalkOptions};
use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
use crate::tier::tier::TierConfigMgr;
use crate::{
bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk},
endpoints::EndpointServerPools,
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
rpc::S3PeerSys,
sets::Sets,
store_api::{GetObjectReader, ListObjectsV2Info, ObjectInfo, ObjectOptions, PutObjReader},
store_init,
};
use futures::future::join_all;
@@ -75,11 +74,14 @@ use rustfs_config::server_config::{Config, get_global_server_config, set_global_
use rustfs_filemeta::FileInfo;
use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
};
use rustfs_storage_api::{
HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
};
use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf};
use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
use std::cmp::Ordering;
@@ -100,6 +102,11 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, instrument, warn};
use uuid::Uuid;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
/// Check if a directory contains any xl.meta files (indicating actual S3 objects)
/// This is used to determine if a bucket is empty for deletion purposes.
async fn has_xlmeta_files(path: &std::path::Path) -> bool {
-160
View File
@@ -1,160 +0,0 @@
use super::*;
use rustfs_storage_api::{
CompletePart, DeletedObject, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, ObjectToDelete,
PartInfo,
};
pub trait ObjectIO:
rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
impl<T> ObjectIO for T where
T: rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
/// Object-level storage operations (beyond basic I/O in ObjectIO).
pub trait ObjectOperations:
rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
impl<T> ObjectOperations for T where
T: rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
/// Listing and walking operations.
pub trait ListOperations:
rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + Send
+ Sync
+ Debug
{
}
impl<T> ListOperations for T where
T: rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + Send
+ Sync
+ Debug
{
}
/// Multipart upload operations.
pub trait MultipartOperations:
rustfs_storage_api::MultipartOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
PutObjectReader = PutObjReader,
CompletePart = CompletePart,
ListMultipartsInfo = ListMultipartsInfo,
MultipartUploadResult = MultipartUploadResult,
PartInfo = PartInfo,
MultipartInfo = MultipartInfo,
ListPartsInfo = ListPartsInfo,
> + Send
+ Sync
+ Debug
{
}
impl<T> MultipartOperations for T where
T: rustfs_storage_api::MultipartOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
PutObjectReader = PutObjReader,
CompletePart = CompletePart,
ListMultipartsInfo = ListMultipartsInfo,
MultipartUploadResult = MultipartUploadResult,
PartInfo = PartInfo,
MultipartInfo = MultipartInfo,
ListPartsInfo = ListPartsInfo,
> + Send
+ Sync
+ Debug
{
}
/// Healing and repair operations.
pub trait HealOperations:
rustfs_storage_api::HealOperations<Error = Error, HealResultItem = HealResultItem, HealOptions = HealOpts> + Send + Sync + Debug
{
}
impl<T> HealOperations for T where
T: rustfs_storage_api::HealOperations<Error = Error, HealResultItem = HealResultItem, HealOptions = HealOpts>
+ Send
+ Sync
+ Debug
{
}
/// Namespace lock operations needed by consumers that only coordinate object
/// mutations but do not require the full storage API surface.
pub trait NamespaceLocking:
rustfs_storage_api::NamespaceLocking<Error = Error, NamespaceLock = NamespaceLockWrapper> + Send + Sync + Debug + 'static
{
}
impl<T> NamespaceLocking for T where
T: rustfs_storage_api::NamespaceLocking<Error = Error, NamespaceLock = NamespaceLockWrapper> + Send + Sync + Debug + 'static
{
}
+13 -3
View File
@@ -21,17 +21,21 @@ use crate::disk::{DiskInfo, DiskStore};
use crate::error::{
Error, Result, StorageError, is_all_not_found, is_all_volume_not_found, is_err_bucket_not_found, to_object_err,
};
use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::set_disk::SetDisks;
use crate::store_api::{ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectInfoOrErr, ObjectOptions, WalkOptions};
use crate::store::ECStore;
use crate::store_utils::is_reserved_or_invalid_bucket;
use crate::{store::ECStore, store_api::ListObjectsV2Info};
use futures::future::join_all;
use rand::seq::SliceRandom;
use rustfs_filemeta::{
MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
merge_file_meta_versions,
};
use rustfs_storage_api::{ObjectOperations as _, VersionMarker, WalkVersionsSortOrder};
use rustfs_storage_api::{
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo as StorageListObjectsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations as _,
VersionMarker, WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
};
use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
@@ -48,6 +52,12 @@ const MAX_OBJECT_LIST: i32 = 1000;
const METACACHE_SHARE_PREFIX: bool = false;
type ListObjectsInfo = StorageListObjectsInfo<ObjectInfo>;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
fn normalize_max_keys(max_keys: i32) -> i32 {
max_keys.min(MAX_OBJECT_LIST)
}
+70 -16
View File
@@ -49,11 +49,13 @@ use crate::{
config::com::{CONFIG_PREFIX, read_config},
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
global::is_first_cluster_node_local,
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations},
store::ECStore,
store_api::{ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
};
use rustfs_filemeta::FileInfo;
use rustfs_rio::HashReader;
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::S3ErrorCode;
@@ -1032,19 +1034,36 @@ impl TierConfigMgr {
self.save_tiering_config(api).await
}
pub async fn save_tiering_config<S: ObjectIO>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
pub async fn save_tiering_config<S>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let data = encode_external_tiering_config_blob(self)?;
let config_file = tier_config_path(TIER_CONFIG_FILE);
self.save_config(api, &config_file, data).await
}
pub async fn save_config<S: ObjectIO>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
) -> std::result::Result<(), std::io::Error> {
pub async fn save_config<S>(&self, api: Arc<S>, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
self.save_config_with_opts(
api,
file,
@@ -1057,13 +1076,24 @@ impl TierConfigMgr {
.await
}
pub async fn save_config_with_opts<S: ObjectIO>(
pub async fn save_config_with_opts<S>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
opts: &ObjectOptions,
) -> std::result::Result<(), std::io::Error> {
) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
debug!("save tier config:{}", file);
let mut put_data = PutObjReader::from_vec(data.to_vec());
let _ = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await?;
@@ -1099,7 +1129,10 @@ impl TierConfigMgr {
}
}
async fn new_and_save_tiering_config<S: ObjectIO>(api: Arc<S>) -> Result<TierConfigMgr> {
async fn new_and_save_tiering_config<S>(api: Arc<S>) -> Result<TierConfigMgr>
where
S: EcstoreObjectIO,
{
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
@@ -1158,12 +1191,15 @@ async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMg
}
}
async fn read_tier_config_from_bucket<S: ObjectIO>(
async fn read_tier_config_from_bucket<S>(
api: Arc<S>,
bucket: &str,
path: &str,
opts: &ObjectOptions,
) -> io::Result<Option<Vec<u8>>> {
) -> io::Result<Option<Vec<u8>>>
where
S: EcstoreObjectIO,
{
let mut rd = match api.get_object_reader(bucket, path, None, HeaderMap::new(), opts).await {
Ok(v) => v,
Err(err) if is_err_config_not_found(&err) => return Ok(None),
@@ -1176,7 +1212,10 @@ async fn read_tier_config_from_bucket<S: ObjectIO>(
Ok(Some(data))
}
async fn write_tier_config_to_rustfs<S: ObjectIO>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()> {
async fn write_tier_config_to_rustfs<S>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()>
where
S: EcstoreObjectIO,
{
let mut put_data = PutObjReader::from_vec(data.to_vec());
api.put_object(
RUSTFS_META_BUCKET,
@@ -1194,7 +1233,22 @@ async fn write_tier_config_to_rustfs<S: ObjectIO>(api: Arc<S>, path: &str, data:
pub async fn try_migrate_tiering_config<S>(api: Arc<S>)
where
S: ObjectIO + ObjectOperations,
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let target_path = tier_config_path(TIER_CONFIG_FILE);
if api
@@ -16,12 +16,25 @@ use rustfs_common::heal_channel::HealOpts;
use rustfs_ecstore::{
disk::DiskStore,
error::Error,
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
store::ECStore,
store_api::{HealOperations, NamespaceLocking},
};
use rustfs_filemeta::FileInfo;
use rustfs_lock::NamespaceLockWrapper;
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::{HealOperations as StorageHealOperations, NamespaceLocking as StorageNamespaceLocking, StorageAdminApi};
use rustfs_storage_api::{
CompletePart, DeletedObject, HTTPRangeSpec, HealOperations as StorageHealOperations, ListMultipartsInfo,
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListPartsInfo,
MultipartInfo, MultipartOperations as StorageMultipartOperations, MultipartUploadResult,
NamespaceLocking as StorageNamespaceLocking, ObjectIO as StorageObjectIO, ObjectInfoOrErr as StorageObjectInfoOrErr,
ObjectOperations as StorageObjectOperations, ObjectToDelete, PartInfo, StorageAdminApi, WalkOptions as StorageWalkOptions,
};
use tokio_util::sync::CancellationToken;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
fn storage_admin_api_type_name<T>() -> &'static str
where
@@ -35,20 +48,6 @@ where
std::any::type_name::<T>()
}
fn namespace_locking_type_name<T>() -> &'static str
where
T: NamespaceLocking,
{
std::any::type_name::<T>()
}
fn heal_operations_type_name<T>() -> &'static str
where
T: HealOperations,
{
std::any::type_name::<T>()
}
fn storage_namespace_locking_type_name<T>() -> &'static str
where
T: StorageNamespaceLocking<Error = Error, NamespaceLock = NamespaceLockWrapper>,
@@ -56,6 +55,68 @@ where
std::any::type_name::<T>()
}
fn storage_object_io_type_name<T>() -> &'static str
where
T: StorageObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
std::any::type_name::<T>()
}
fn storage_object_operations_type_name<T>() -> &'static str
where
T: StorageObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
std::any::type_name::<T>()
}
fn storage_list_operations_type_name<T>() -> &'static str
where
T: rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
>,
{
std::any::type_name::<T>()
}
fn storage_multipart_operations_type_name<T>() -> &'static str
where
T: StorageMultipartOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
PutObjectReader = PutObjReader,
CompletePart = CompletePart,
ListMultipartsInfo = ListMultipartsInfo,
MultipartUploadResult = MultipartUploadResult,
PartInfo = PartInfo,
MultipartInfo = MultipartInfo,
ListPartsInfo = ListPartsInfo,
>,
{
std::any::type_name::<T>()
}
fn storage_heal_operations_type_name<T>() -> &'static str
where
T: StorageHealOperations<Error = Error, HealResultItem = HealResultItem, HealOptions = HealOpts>,
@@ -68,21 +129,31 @@ fn ecstore_implements_storage_admin_api_contract() {
assert!(storage_admin_api_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_namespace_locking_contract() {
assert!(namespace_locking_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_heal_operations_contract() {
assert!(heal_operations_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_storage_namespace_locking_contract() {
assert!(storage_namespace_locking_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_storage_object_io_contract() {
assert!(storage_object_io_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_storage_object_operations_contract() {
assert!(storage_object_operations_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_storage_list_operations_contract() {
assert!(storage_list_operations_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_storage_multipart_operations_contract() {
assert!(storage_multipart_operations_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_storage_heal_operations_contract() {
assert!(storage_heal_operations_type_name::<ECStore>().ends_with("::ECStore"));
@@ -8,7 +8,7 @@ use rustfs_ecstore::bitrot::create_bitrot_reader;
use rustfs_ecstore::disk::endpoint::Endpoint;
use rustfs_ecstore::disk::{DiskAPI as _, DiskOption, new_disk};
use rustfs_ecstore::erasure_coding::Erasure;
use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use rustfs_ecstore::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
use serde::Deserialize;
use sha2::{Digest, Sha256};
+3 -3
View File
@@ -29,6 +29,6 @@ pub(crate) use rustfs_ecstore::global::GLOBAL_LOCAL_DISK_MAP;
#[cfg(test)]
pub(crate) use rustfs_ecstore::disk::{DiskOption, new_disk};
pub type HealObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub type HealObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub type HealPutObjReader = rustfs_ecstore::store_api::PutObjReader;
pub type HealObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
pub type HealObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
pub type HealPutObjReader = rustfs_ecstore::object_api::PutObjReader;
+2 -2
View File
@@ -17,8 +17,8 @@ use std::sync::Arc;
pub(crate) const IAM_CONFIG_ROOT_PREFIX: &str = rustfs_ecstore::config::RUSTFS_CONFIG_PREFIX;
pub(crate) type IamEcstoreError = rustfs_ecstore::error::Error;
pub(crate) type IamConfigObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub(crate) type IamConfigObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub(crate) type IamConfigObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
pub(crate) type IamConfigObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
pub(crate) type IamStorageError = rustfs_ecstore::error::StorageError;
pub(crate) type IamStorageResult<T> = rustfs_ecstore::error::Result<T>;
pub(crate) type IamStore = rustfs_ecstore::store::ECStore;
+2 -2
View File
@@ -12,5 +12,5 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(super) type IamObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub(super) type IamObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub(super) type IamObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
pub(super) type IamObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
+1 -1
View File
@@ -18,7 +18,7 @@ use rustfs_ecstore::{config, global};
use crate::event::NotifyObjectInfo;
type EcstoreObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
type EcstoreObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
#[derive(Debug)]
pub(crate) enum NotifyConfigStoreError {
+4 -4
View File
@@ -18,10 +18,10 @@ pub(super) type SwiftBucketMetadata = rustfs_ecstore::bucket::metadata::BucketMe
pub(super) type SwiftStorageResult<T> = rustfs_ecstore::error::Result<T>;
pub(super) type SwiftStore = rustfs_ecstore::store::ECStore;
pub type SwiftGetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
pub type SwiftObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub type SwiftObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub type SwiftPutObjReader = rustfs_ecstore::store_api::PutObjReader;
pub type SwiftGetObjectReader = rustfs_ecstore::object_api::GetObjectReader;
pub type SwiftObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
pub type SwiftObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
pub type SwiftPutObjReader = rustfs_ecstore::object_api::PutObjReader;
pub fn resolve_swift_object_store_handle() -> Option<Arc<SwiftStore>> {
rustfs_ecstore::resolve_object_store_handle()
+3 -3
View File
@@ -12,9 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) type SelectGetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
pub(crate) type SelectObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub(crate) type SelectObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub(crate) type SelectGetObjectReader = rustfs_ecstore::object_api::GetObjectReader;
pub(crate) type SelectObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
pub(crate) type SelectObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
pub(crate) type SelectStorageError = rustfs_ecstore::error::StorageError;
pub(crate) type SelectStore = rustfs_ecstore::store::ECStore;
+4 -4
View File
@@ -58,11 +58,11 @@ pub(crate) use rustfs_ecstore::config::init as init_ecstore_config_for_scanner_t
#[cfg(test)]
pub(crate) use rustfs_ecstore::disk::{DiskOption, endpoint::Endpoint, new_disk};
pub type ScannerGetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
pub type ScannerObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub type ScannerObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub type ScannerGetObjectReader = rustfs_ecstore::object_api::GetObjectReader;
pub type ScannerObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
pub type ScannerObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
pub type ScannerObjectToDelete = ObjectToDelete;
pub type ScannerPutObjReader = rustfs_ecstore::store_api::PutObjReader;
pub type ScannerPutObjReader = rustfs_ecstore::object_api::PutObjReader;
pub(crate) fn resolve_scanner_object_store_handle() -> Option<Arc<ECStore>> {
rustfs_ecstore::resolve_object_store_handle()