mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
refactor: remove ecstore operation compat facades (#3608)
This commit is contained in:
@@ -35,9 +35,9 @@ use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_ver
|
||||
use crate::event_notification::{EventArgs, send_event};
|
||||
use crate::global::GLOBAL_LocalNodeName;
|
||||
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks};
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use crate::tier::warm_backend::WarmBackendGetOpts;
|
||||
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
|
||||
use futures::Future;
|
||||
@@ -2863,9 +2863,9 @@ mod tests {
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::error::is_err_invalid_upload_id;
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use futures::FutureExt;
|
||||
use rustfs_common::metrics::{IlmAction, global_metrics};
|
||||
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
|
||||
|
||||
@@ -26,7 +26,7 @@ use time::{self, Duration, OffsetDateTime};
|
||||
use tracing::debug;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::store_api::ObjectInfo;
|
||||
use crate::object_api::ObjectInfo;
|
||||
|
||||
pub const TRANSITION_COMPLETE: &str = "complete";
|
||||
pub const TRANSITION_PENDING: &str = "pending";
|
||||
|
||||
@@ -23,9 +23,10 @@ use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_t
|
||||
use crate::config::com::{delete_config, read_config, save_config};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{ObjectIO, ObjectOperations};
|
||||
use rustfs_storage_api::ListOperations as _;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ListOperations as _, ObjectIO, ObjectOperations, ObjectToDelete};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
|
||||
@@ -104,7 +105,15 @@ fn encode_tier_delete_journal_entry(je: &Jentry) -> Result<Vec<u8>> {
|
||||
|
||||
pub async fn persist_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||
where
|
||||
S: ObjectIO,
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = http::HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?;
|
||||
save_config(api, &tier_delete_journal_object_name(je), data)
|
||||
@@ -114,7 +123,14 @@ where
|
||||
|
||||
pub async fn remove_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
|
||||
where
|
||||
S: ObjectOperations,
|
||||
S: ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
match delete_config(api, &tier_delete_journal_object_name(je)).await {
|
||||
Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
|
||||
|
||||
@@ -19,13 +19,20 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::Result;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{ObjectInfo, ObjectInfoOrErr, WalkOptions};
|
||||
use rustfs_storage_api::{BucketOperations, BucketOptions, ListOperations as _};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_storage_api::{
|
||||
BucketOperations, BucketOptions, ListOperations as _, ObjectInfoOrErr as StorageObjectInfoOrErr,
|
||||
WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
|
||||
pub const DEFAULT_FREE_VERSION_RECOVERY_LIMIT: usize = 1_000;
|
||||
const DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT: usize = 10_000;
|
||||
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, crate::error::Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct FreeVersionRecoveryStats {
|
||||
pub scanned: usize,
|
||||
|
||||
@@ -17,11 +17,18 @@
|
||||
use crate::bucket::metadata::BUCKET_METADATA_FILE;
|
||||
use crate::bucket::replication::{decode_resync_file, encode_resync_file};
|
||||
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
||||
use crate::store_api::{ListOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader};
|
||||
use crate::error::Error;
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations};
|
||||
use http::HeaderMap;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use rustfs_policy::policy::PolicyDoc;
|
||||
use rustfs_storage_api::{BucketOperations, BucketOptions};
|
||||
use rustfs_storage_api::{
|
||||
BucketOperations, BucketOptions, DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectIO, ObjectInfoOrErr as StorageObjectInfoOrErr,
|
||||
ObjectOperations, ObjectToDelete, WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
@@ -40,6 +47,11 @@ const IAM_POLICY_DB_PREFIX: &str = "config/iam/policydb/";
|
||||
const REPLICATION_META_DIR: &str = ".replication";
|
||||
const RESYNC_META_FILE: &str = "resync.bin";
|
||||
|
||||
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct CompatIamFormat {
|
||||
#[serde(default)]
|
||||
@@ -179,7 +191,23 @@ fn normalize_bucket_meta_blob(path: &str, data: &[u8]) -> std::result::Result<Op
|
||||
/// Skips buckets that already exist in RustFS (idempotent).
|
||||
pub async fn try_migrate_bucket_metadata<S>(store: Arc<S>)
|
||||
where
|
||||
S: BucketOperations<Error = crate::error::Error> + ObjectIO + ObjectOperations,
|
||||
S: BucketOperations<Error = crate::error::Error>
|
||||
+ ObjectIO<
|
||||
Error = crate::error::Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + ObjectOperations<
|
||||
Error = crate::error::Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
let buckets_list = match store
|
||||
.list_bucket(&BucketOptions {
|
||||
@@ -224,7 +252,7 @@ where
|
||||
|
||||
async fn migrate_one_if_missing<S>(store: Arc<S>, opts: &ObjectOptions, headers: &HeaderMap, path: &str, label: &str)
|
||||
where
|
||||
S: ObjectIO + ObjectOperations,
|
||||
S: EcstoreObjectIO + EcstoreObjectOperations,
|
||||
{
|
||||
if store
|
||||
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
|
||||
@@ -278,7 +306,30 @@ where
|
||||
/// If list_objects_v2 on the legacy bucket fails (e.g. format differs), migration is skipped.
|
||||
pub async fn try_migrate_iam_config<S>(store: Arc<S>)
|
||||
where
|
||||
S: ListOperations + ObjectIO + ObjectOperations,
|
||||
S: ListOperations<
|
||||
Error = crate::error::Error,
|
||||
ListObjectsV2Info = ListObjectsV2Info,
|
||||
ListObjectVersionsInfo = ListObjectVersionsInfo,
|
||||
ObjectInfoOrErr = ObjectInfoOrErr,
|
||||
WalkOptions = WalkOptions,
|
||||
WalkCancellation = tokio_util::sync::CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
|
||||
> + ObjectIO<
|
||||
Error = crate::error::Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + ObjectOperations<
|
||||
Error = crate::error::Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
let opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::bucket::metadata_sys::get_object_lock_config;
|
||||
use crate::bucket::object_lock::objectlock;
|
||||
use crate::store_api::ObjectInfo;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use s3s::dto::{DefaultRetention, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
|
||||
use std::sync::Arc;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
@@ -27,7 +27,8 @@ use crate::bucket::replication::replication_state::ReplicationStats;
|
||||
use crate::config::com::{read_config, save_config};
|
||||
use crate::disk::BUCKET_META_PREFIX;
|
||||
use crate::error::Error as EcstoreError;
|
||||
use crate::store_api::{ObjectIO, ObjectInfo, ObjectOptions};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::storage_api_contracts::EcstoreObjectIO;
|
||||
use lazy_static::lazy_static;
|
||||
use rustfs_filemeta::MrfOpKind;
|
||||
use rustfs_filemeta::MrfReplicateEntry;
|
||||
@@ -1252,7 +1253,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
/// Returns `true` on success; on failure logs the error and returns `false`.
|
||||
/// Callers must NOT clear their in-memory buffer on `false` so the next tick
|
||||
/// can retry — otherwise a transient storage error permanently drops the batch.
|
||||
async fn flush_mrf_to_disk<S: ObjectIO>(entries: &[MrfReplicateEntry], storage: &Arc<S>) -> bool {
|
||||
async fn flush_mrf_to_disk<S: EcstoreObjectIO>(entries: &[MrfReplicateEntry], storage: &Arc<S>) -> bool {
|
||||
match encode_mrf_file(entries) {
|
||||
Ok(data) => {
|
||||
if let Err(e) = save_config(storage.clone(), MRF_REPLICATION_FILE, data).await {
|
||||
@@ -1281,7 +1282,7 @@ async fn flush_mrf_to_disk<S: ObjectIO>(entries: &[MrfReplicateEntry], storage:
|
||||
}
|
||||
|
||||
/// Load bucket resync metadata from disk
|
||||
async fn load_bucket_resync_metadata<S: ObjectIO>(
|
||||
async fn load_bucket_resync_metadata<S: EcstoreObjectIO>(
|
||||
bucket: &str,
|
||||
obj_api: Arc<S>,
|
||||
) -> Result<BucketReplicationResyncStatus, EcstoreError> {
|
||||
|
||||
@@ -31,9 +31,10 @@ use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_fo
|
||||
use crate::event_notification::{EventArgs, send_event};
|
||||
use crate::global::GLOBAL_LocalNodeName;
|
||||
use crate::global::get_global_bucket_monitor;
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::resolve_object_store_handle;
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::store_api::{ListOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, WalkOptions};
|
||||
use crate::storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations};
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
@@ -52,13 +53,17 @@ use http_body_util::StreamBody;
|
||||
use regex::Regex;
|
||||
use rmp_serde;
|
||||
use rustfs_filemeta::{
|
||||
MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo,
|
||||
FileInfo, MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo,
|
||||
ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType,
|
||||
ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType,
|
||||
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectToDelete};
|
||||
use rustfs_storage_api::{
|
||||
DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, NamespaceLocking as StorageNamespaceLocking, ObjectIO,
|
||||
ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations, ObjectToDelete, WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
|
||||
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
|
||||
@@ -102,10 +107,67 @@ const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed";
|
||||
const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed";
|
||||
const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed";
|
||||
|
||||
/// Storage capabilities required by bucket replication workers.
|
||||
pub trait ReplicationStorage: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking {}
|
||||
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
impl<T> ReplicationStorage for T where T: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking {}
|
||||
/// Storage capabilities required by bucket replication workers.
|
||||
pub trait ReplicationStorage:
|
||||
ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
> + ListOperations<
|
||||
Error = Error,
|
||||
ListObjectsV2Info = ListObjectsV2Info,
|
||||
ListObjectVersionsInfo = ListObjectVersionsInfo,
|
||||
ObjectInfoOrErr = ObjectInfoOrErr,
|
||||
WalkOptions = WalkOptions,
|
||||
WalkCancellation = CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
|
||||
> + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> ReplicationStorage for T where
|
||||
T: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
> + ListOperations<
|
||||
Error = Error,
|
||||
ListObjectsV2Info = ListObjectsV2Info,
|
||||
ListObjectVersionsInfo = ListObjectVersionsInfo,
|
||||
ObjectInfoOrErr = ObjectInfoOrErr,
|
||||
WalkOptions = WalkOptions,
|
||||
WalkCancellation = CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
|
||||
> + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
{
|
||||
}
|
||||
|
||||
pub(crate) const REPLICATION_DIR: &str = ".replication";
|
||||
pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
|
||||
@@ -528,7 +590,18 @@ impl ReplicationResyncer {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn mark_status<S: ObjectIO>(&self, status: ResyncStatusType, opts: ResyncOpts, obj_layer: Arc<S>) -> Result<()> {
|
||||
pub async fn mark_status<S>(&self, status: ResyncStatusType, opts: ResyncOpts, obj_layer: Arc<S>) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
let bucket_status = {
|
||||
let mut status_map = self.status_map.write().await;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
@@ -641,7 +714,18 @@ impl ReplicationResyncer {
|
||||
bucket_status.last_update = Some(now);
|
||||
}
|
||||
|
||||
pub async fn persist_to_disk<S: ObjectIO>(&self, cancel_token: CancellationToken, api: Arc<S>) {
|
||||
pub async fn persist_to_disk<S>(&self, cancel_token: CancellationToken, api: Arc<S>)
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
let mut interval = tokio::time::interval(RESYNC_TIME_INTERVAL);
|
||||
|
||||
let mut last_update_times = HashMap::new();
|
||||
@@ -694,7 +778,7 @@ impl ReplicationResyncer {
|
||||
}
|
||||
}
|
||||
|
||||
async fn resync_bucket_mark_status<S: ObjectIO>(&self, status: ResyncStatusType, opts: ResyncOpts, storage: Arc<S>) {
|
||||
async fn resync_bucket_mark_status<S: EcstoreObjectIO>(&self, status: ResyncStatusType, opts: ResyncOpts, storage: Arc<S>) {
|
||||
if let Err(err) = self.mark_status(status, opts.clone(), storage.clone()).await {
|
||||
error!(
|
||||
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
|
||||
@@ -1219,7 +1303,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn save_resync_status<S: ObjectIO>(
|
||||
pub(crate) async fn save_resync_status<S: EcstoreObjectIO>(
|
||||
bucket: &str,
|
||||
status: &BucketReplicationResyncStatus,
|
||||
api: Arc<S>,
|
||||
@@ -2181,7 +2265,7 @@ pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicat
|
||||
}
|
||||
}
|
||||
|
||||
async fn source_delete_marker_missing<S: ObjectOperations>(
|
||||
async fn source_delete_marker_missing<S: EcstoreObjectOperations>(
|
||||
storage: &S,
|
||||
bucket: &str,
|
||||
object_name: &str,
|
||||
@@ -2873,13 +2957,13 @@ pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, s
|
||||
}
|
||||
|
||||
trait ReplicateObjectInfoExt {
|
||||
async fn replicate_object<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
|
||||
async fn replicate_all<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
|
||||
async fn replicate_object<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
|
||||
async fn replicate_all<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
|
||||
fn to_object_info(&self) -> ObjectInfo;
|
||||
}
|
||||
|
||||
impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
async fn replicate_object<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
|
||||
async fn replicate_object<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
|
||||
let bucket = self.bucket.clone();
|
||||
let object = self.name.clone();
|
||||
|
||||
@@ -3174,7 +3258,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
rinfo
|
||||
}
|
||||
|
||||
async fn replicate_all<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
|
||||
async fn replicate_all<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
|
||||
let start_time = OffsetDateTime::now_utc();
|
||||
|
||||
let bucket = self.bucket.clone();
|
||||
@@ -3896,7 +3980,7 @@ fn part_range_spec_from_actual_size(offset: i64, part_size: i64) -> std::io::Res
|
||||
))
|
||||
}
|
||||
|
||||
struct MultipartReplicationContext<'a, S: ObjectIO> {
|
||||
struct MultipartReplicationContext<'a, S: EcstoreObjectIO> {
|
||||
storage: Arc<S>,
|
||||
cli: Arc<TargetClient>,
|
||||
src_bucket: &'a str,
|
||||
@@ -3908,7 +3992,7 @@ struct MultipartReplicationContext<'a, S: ObjectIO> {
|
||||
put_opts: PutObjectOptions,
|
||||
}
|
||||
|
||||
async fn replicate_object_with_multipart<S: ObjectIO>(ctx: MultipartReplicationContext<'_, S>) -> std::io::Result<()> {
|
||||
async fn replicate_object_with_multipart<S: EcstoreObjectIO>(ctx: MultipartReplicationContext<'_, S>) -> std::io::Result<()> {
|
||||
let MultipartReplicationContext {
|
||||
storage,
|
||||
cli,
|
||||
|
||||
@@ -44,7 +44,7 @@ use crate::client::{
|
||||
};
|
||||
use crate::{
|
||||
disk::DiskAPI,
|
||||
store_api::{GetObjectReader, ObjectInfo},
|
||||
object_api::{GetObjectReader, ObjectInfo},
|
||||
};
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ use std::collections::HashMap;
|
||||
use crate::client::utils::base64_decode;
|
||||
use crate::client::utils::base64_encode;
|
||||
use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
|
||||
use crate::{disk::DiskAPI, store_api::GetObjectReader};
|
||||
use crate::{disk::DiskAPI, object_api::GetObjectReader};
|
||||
use s3s::header::{
|
||||
X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
|
||||
};
|
||||
|
||||
@@ -25,7 +25,7 @@ use std::{collections::HashMap, io::Cursor, sync::Arc};
|
||||
use tokio::io::BufReader;
|
||||
|
||||
use crate::error::ErrorResponse;
|
||||
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use rustfs_filemeta::ObjectPartInfo;
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
|
||||
@@ -24,8 +24,8 @@ use crate::bucket::lifecycle::lifecycle;
|
||||
use crate::bucket::replication::{DeletedObjectReplicationInfo, check_replicate_delete, schedule_replication_delete};
|
||||
use crate::bucket::versioning::VersioningApi;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::ObjectOptions;
|
||||
use rustfs_filemeta::{REPLICATE_INCOMING_DELETE, ReplicationState, version_purge_statuses_map};
|
||||
use rustfs_lock::MAX_DELETE_LIST;
|
||||
use rustfs_storage_api::{ObjectOperations as _, ObjectToDelete};
|
||||
|
||||
@@ -33,7 +33,7 @@ use crate::client::{
|
||||
credentials::{CredContext, Credentials, SignatureType, Static},
|
||||
signer_error,
|
||||
};
|
||||
use crate::{client::checksum::ChecksumMode, store_api::GetObjectReader};
|
||||
use crate::{client::checksum::ChecksumMode, object_api::GetObjectReader};
|
||||
use futures::{Future, StreamExt};
|
||||
use http::{HeaderMap, HeaderName};
|
||||
use http::{
|
||||
|
||||
@@ -16,7 +16,8 @@ use crate::config::{audit, notify, oidc, set_global_storage_class, storageclass}
|
||||
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::global::is_first_cluster_node_local;
|
||||
use crate::store_api::{ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, PutObjReader};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::storage_api_contracts::EcstoreObjectIO;
|
||||
use http::HeaderMap;
|
||||
use rustfs_config::audit::{
|
||||
AUDIT_AMQP_KEYS, AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS,
|
||||
@@ -32,7 +33,8 @@ use rustfs_config::notify::{
|
||||
use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
|
||||
use rustfs_config::server_config::{Config, KVS};
|
||||
use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
|
||||
use rustfs_storage_api::StorageAdminApi;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete, StorageAdminApi};
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use serde_json::{Map, Value};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
@@ -182,12 +184,34 @@ fn audit_target_descriptors() -> [TargetConfigDescriptor; 9] {
|
||||
}
|
||||
|
||||
#[instrument(skip(api))]
|
||||
pub async fn read_config<S: ObjectIO>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
|
||||
pub async fn read_config<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub async fn read_config_no_lock<S: ObjectIO>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
|
||||
pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
let (data, _obj) = read_config_with_metadata(
|
||||
api,
|
||||
file,
|
||||
@@ -200,11 +224,18 @@ pub async fn read_config_no_lock<S: ObjectIO>(api: Arc<S>, file: &str) -> Result
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub async fn read_config_with_metadata<S: ObjectIO>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)> {
|
||||
pub async fn read_config_with_metadata<S>(api: Arc<S>, file: &str, opts: &ObjectOptions) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
let h = HeaderMap::new();
|
||||
let mut rd = api
|
||||
.get_object_reader(RUSTFS_META_BUCKET, file, None, h, opts)
|
||||
@@ -228,7 +259,18 @@ pub async fn read_config_with_metadata<S: ObjectIO>(
|
||||
}
|
||||
|
||||
#[instrument(skip(api, data))]
|
||||
pub async fn save_config<S: ObjectIO>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
|
||||
pub async fn save_config<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
save_config_with_opts(
|
||||
api,
|
||||
file,
|
||||
@@ -242,7 +284,17 @@ pub async fn save_config<S: ObjectIO>(api: Arc<S>, file: &str, data: Vec<u8>) ->
|
||||
}
|
||||
|
||||
#[instrument(skip(api))]
|
||||
pub async fn delete_config<S: ObjectOperations>(api: Arc<S>, file: &str) -> Result<()> {
|
||||
pub async fn delete_config<S>(api: Arc<S>, file: &str) -> Result<()>
|
||||
where
|
||||
S: ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
match api
|
||||
.delete_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
@@ -266,7 +318,18 @@ pub async fn delete_config<S: ObjectOperations>(api: Arc<S>, file: &str) -> Resu
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn save_config_with_opts<S: ObjectIO>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
|
||||
pub async fn save_config_with_opts<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
let mut put_data = PutObjReader::from_vec(data);
|
||||
if let Err(err) = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
|
||||
error!("save_config_with_opts: err: {:?}, file: {}", err, file);
|
||||
@@ -281,7 +344,7 @@ fn new_server_config() -> Config {
|
||||
|
||||
async fn new_and_save_server_config<S>(api: Arc<S>) -> Result<Config>
|
||||
where
|
||||
S: ObjectIO + StorageAdminApi,
|
||||
S: EcstoreObjectIO + StorageAdminApi,
|
||||
{
|
||||
let mut cfg = new_server_config();
|
||||
lookup_configs(&mut cfg, api.clone()).await;
|
||||
@@ -985,7 +1048,22 @@ fn is_object_not_found(err: &Error) -> bool {
|
||||
|
||||
pub async fn try_migrate_server_config<S>(api: Arc<S>)
|
||||
where
|
||||
S: ObjectIO + ObjectOperations,
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
let config_file = get_config_file();
|
||||
match api
|
||||
@@ -1069,7 +1147,7 @@ where
|
||||
/// Handle the situation where the configuration file does not exist, create and save a new configuration
|
||||
async fn handle_missing_config<S>(api: Arc<S>, context: &str) -> Result<Config>
|
||||
where
|
||||
S: ObjectIO + StorageAdminApi,
|
||||
S: EcstoreObjectIO + StorageAdminApi,
|
||||
{
|
||||
warn!("Configuration not found ({}): Start initializing new configuration", context);
|
||||
let cfg = if is_first_cluster_node_local().await {
|
||||
@@ -1091,7 +1169,15 @@ fn handle_config_read_error(err: Error, file_path: &str) -> Result<Config> {
|
||||
|
||||
pub async fn read_config_without_migrate<S>(api: Arc<S>) -> Result<Config>
|
||||
where
|
||||
S: ObjectIO + StorageAdminApi,
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + StorageAdminApi,
|
||||
{
|
||||
let config_file = get_config_file();
|
||||
|
||||
@@ -1105,7 +1191,7 @@ where
|
||||
|
||||
async fn read_server_config<S>(api: Arc<S>, data: &[u8]) -> Result<Config>
|
||||
where
|
||||
S: ObjectIO + StorageAdminApi,
|
||||
S: EcstoreObjectIO + StorageAdminApi,
|
||||
{
|
||||
// If the provided data is empty, try to read from the file again
|
||||
if data.is_empty() {
|
||||
@@ -1129,7 +1215,18 @@ where
|
||||
Ok(cfg.merge())
|
||||
}
|
||||
|
||||
pub async fn save_server_config<S: ObjectIO>(api: Arc<S>, cfg: &Config) -> Result<()> {
|
||||
pub async fn save_server_config<S>(api: Arc<S>, cfg: &Config) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
let config_file = get_config_file();
|
||||
let existing = match read_config(api.clone(), &config_file).await {
|
||||
Ok(v) => Some(v),
|
||||
@@ -1216,8 +1313,8 @@ mod tests {
|
||||
use crate::endpoints::SetupType;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use http::HeaderMap;
|
||||
use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
|
||||
use rustfs_config::notify::{
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::{Error, Result, is_err_data_movement_overwrite, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use bytes::Bytes;
|
||||
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
|
||||
use rustfs_storage_api::{CompletePart, MultipartOperations as _, ObjectIO as _, ObjectOperations as _};
|
||||
|
||||
@@ -715,7 +715,7 @@ pub fn cache_to_data_usage_info(cache: &DataUsageCache, path: &str, buckets: &[r
|
||||
// Helper functions for DataUsageCache operations
|
||||
pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str) -> crate::error::Result<DataUsageCache> {
|
||||
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
use crate::store_api::ObjectOptions;
|
||||
use crate::object_api::ObjectOptions;
|
||||
use http::HeaderMap;
|
||||
use rand::RngExt;
|
||||
use std::path::Path;
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
use crate::bucket::metadata::BucketMetadata;
|
||||
// use crate::event::name::EventName;
|
||||
use crate::event::targetlist::TargetList;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::ObjectInfo;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
@@ -32,14 +32,15 @@ pub mod error;
|
||||
pub mod global;
|
||||
pub mod metrics_realtime;
|
||||
pub mod notification_sys;
|
||||
pub mod object_api;
|
||||
pub mod pools;
|
||||
pub mod rebalance;
|
||||
pub mod rio;
|
||||
pub mod rpc;
|
||||
pub mod set_disk;
|
||||
mod sets;
|
||||
mod storage_api_contracts;
|
||||
pub mod store;
|
||||
pub mod store_api;
|
||||
mod store_init;
|
||||
pub mod store_list_objects;
|
||||
pub mod store_utils;
|
||||
|
||||
@@ -21,14 +21,11 @@ use crate::store_utils::clean_metadata;
|
||||
use crate::{bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, bucket::lifecycle::lifecycle::TransitionOptions};
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use rustfs_common::heal_channel::HealOpts;
|
||||
use rustfs_filemeta::{
|
||||
FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, ReplicateDecision, ReplicationState, ReplicationStatusType,
|
||||
RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
|
||||
version_purge_statuses_map,
|
||||
};
|
||||
use rustfs_lock::NamespaceLockWrapper;
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_rio::Checksum;
|
||||
use rustfs_storage_api::{ExpirationOptions, HTTPRangeSpec, TransitionedObject};
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
@@ -43,7 +40,6 @@ use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -51,9 +47,7 @@ pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
|
||||
pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
|
||||
|
||||
mod readers;
|
||||
mod traits;
|
||||
mod types;
|
||||
|
||||
pub use readers::*;
|
||||
pub use traits::*;
|
||||
pub use types::*;
|
||||
@@ -4,14 +4,6 @@ use rustfs_storage_api::{
|
||||
VersionMarker,
|
||||
};
|
||||
|
||||
pub type ListObjectsInfo = rustfs_storage_api::ListObjectsInfo<ObjectInfo>;
|
||||
pub type ListObjectsV2Info = rustfs_storage_api::ListObjectsV2Info<ObjectInfo>;
|
||||
pub type ListObjectVersionsInfo = rustfs_storage_api::ListObjectVersionsInfo<ObjectInfo>;
|
||||
pub type ObjectInfoOrErr = rustfs_storage_api::ObjectInfoOrErr<ObjectInfo, Error>;
|
||||
pub type WalkOptions = rustfs_storage_api::WalkOptions<WalkFilter>;
|
||||
pub type ObjectToDelete = rustfs_storage_api::ObjectToDelete;
|
||||
pub type DeletedObject = rustfs_storage_api::DeletedObject;
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
// Use the maximum parity (N/2), used when saving server configuration files
|
||||
@@ -432,7 +424,7 @@ impl ObjectInfo {
|
||||
restore_expires = restore_status.expiry();
|
||||
}
|
||||
|
||||
// Convert parts from rustfs_filemeta::ObjectPartInfo to store_api::ObjectPartInfo
|
||||
// Convert parts from rustfs_filemeta::ObjectPartInfo to object_api::ObjectPartInfo
|
||||
let parts = fi
|
||||
.parts
|
||||
.iter()
|
||||
@@ -715,9 +707,6 @@ fn versions_after_marker(file_infos: &rustfs_filemeta::FileInfoVersions, marker:
|
||||
.map(|idx| &file_infos.versions[idx + 1..])
|
||||
.unwrap_or(&file_infos.versions)
|
||||
}
|
||||
|
||||
type WalkFilter = fn(&FileInfo) -> bool;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -36,9 +36,9 @@ use crate::error::{
|
||||
is_err_operation_canceled, is_err_version_not_found,
|
||||
};
|
||||
use crate::notification_sys::get_global_notification_sys;
|
||||
use crate::object_api::{GetObjectReader, ObjectOptions};
|
||||
use crate::resolve_object_store_handle;
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::store_api::{GetObjectReader, ObjectOptions};
|
||||
use crate::{global::GLOBAL_LifecycleSys, sets::Sets, store::ECStore};
|
||||
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
|
||||
use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered};
|
||||
@@ -1348,7 +1348,7 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
|
||||
};
|
||||
|
||||
let versioned = BucketVersioningSys::prefix_enabled(bucket, &version.name).await;
|
||||
let object_info = crate::store_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
|
||||
let object_info = crate::object_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
|
||||
let event = eval_action_from_lifecycle(lifecycle_config, lock_retention, replication_config, &object_info).await;
|
||||
|
||||
match event.action {
|
||||
|
||||
@@ -20,14 +20,17 @@ use crate::disk::error::DiskError;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::error::{is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found, is_network_or_host_down};
|
||||
use crate::global::get_global_endpoints;
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::pools::ListCallback;
|
||||
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
|
||||
use crate::storage_api_contracts::EcstoreObjectIO;
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOptions};
|
||||
use http::HeaderMap;
|
||||
use rand::RngExt as _;
|
||||
use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
|
||||
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _, StorageAdminApi};
|
||||
use rustfs_storage_api::{
|
||||
HTTPRangeSpec, NamespaceLocking as StorageNamespaceLocking, ObjectIO, ObjectOperations as _, StorageAdminApi,
|
||||
};
|
||||
use rustfs_utils::path::encode_dir_object;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashSet;
|
||||
@@ -589,11 +592,33 @@ impl RebalanceMeta {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
pub async fn load<S: ObjectIO>(&mut self, store: Arc<S>) -> Result<()> {
|
||||
pub async fn load<S>(&mut self, store: Arc<S>) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
self.load_with_opts(store, ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
pub async fn load_with_opts<S: ObjectIO>(&mut self, store: Arc<S>, opts: ObjectOptions) -> Result<()> {
|
||||
pub async fn load_with_opts<S>(&mut self, store: Arc<S>, opts: ObjectOptions) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
let (data, _) = read_config_with_metadata(store, REBAL_META_NAME, &opts).await?;
|
||||
if data.is_empty() {
|
||||
debug!(
|
||||
@@ -634,11 +659,33 @@ impl RebalanceMeta {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn save<S: ObjectIO>(&self, store: Arc<S>) -> Result<()> {
|
||||
pub async fn save<S>(&self, store: Arc<S>) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
self.save_with_opts(store, ObjectOptions::default()).await
|
||||
}
|
||||
|
||||
pub async fn save_with_opts<S: ObjectIO>(&self, store: Arc<S>, opts: ObjectOptions) -> Result<()> {
|
||||
pub async fn save_with_opts<S>(&self, store: Arc<S>, opts: ObjectOptions) -> Result<()>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
if self.pool_stats.is_empty() {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
@@ -667,12 +714,10 @@ impl RebalanceMeta {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
async fn save_rebalance_meta_with_merge<S: ObjectIO + NamespaceLocking>(
|
||||
&self,
|
||||
pool: Arc<S>,
|
||||
local_snapshot: &RebalanceMeta,
|
||||
stage: &str,
|
||||
) -> Result<()> {
|
||||
async fn save_rebalance_meta_with_merge<S>(&self, pool: Arc<S>, local_snapshot: &RebalanceMeta, stage: &str) -> Result<()>
|
||||
where
|
||||
S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
|
||||
let _guard = ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
|
||||
@@ -33,9 +33,7 @@ use crate::erasure_coding;
|
||||
use crate::error::{Error, Result, is_err_version_not_found};
|
||||
use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
|
||||
use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr};
|
||||
use crate::store_api::ListObjectVersionsInfo;
|
||||
use crate::store_api::ObjectOptions;
|
||||
use crate::store_api::{ObjectInfoOrErr, WalkOptions};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::{
|
||||
bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
LifecycleOps, gen_transition_objname, get_transitioned_object_reader, put_restore_opts,
|
||||
@@ -51,7 +49,7 @@ use crate::{
|
||||
// event::name::EventName,
|
||||
event_notification::{EventArgs, send_event},
|
||||
global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure},
|
||||
store_api::{GetObjectReader, ListObjectsV2Info, MultipartOperations, ObjectIO, ObjectInfo, ObjectOperations, PutObjReader},
|
||||
object_api::{GetObjectReader, ObjectInfo, PutObjReader},
|
||||
store_init::load_format_erasure,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
@@ -82,11 +80,14 @@ use rustfs_object_capacity::capacity_scope::{
|
||||
CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope,
|
||||
};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
use rustfs_storage_api::{
|
||||
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
|
||||
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
|
||||
};
|
||||
use rustfs_storage_api::{
|
||||
HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
use rustfs_storage_api::{MultipartOperations as _, NamespaceLocking as _, ObjectIO as _, ObjectOperations as _};
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
|
||||
@@ -131,6 +132,11 @@ use tracing::error;
|
||||
use tracing::{Instrument, debug, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_SET_DISK: &str = "set_disk";
|
||||
const EVENT_SET_DISK_MULTIPART: &str = "set_disk_multipart";
|
||||
@@ -5014,7 +5020,7 @@ mod tests {
|
||||
use crate::disk::health_state::RuntimeDriveHealthState;
|
||||
use crate::endpoints::SetupType;
|
||||
use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
|
||||
use crate::store_api::ObjectInfo;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use crate::store_init::save_format_file;
|
||||
use crate::store_list_objects::ListPathOptions;
|
||||
use rustfs_filemeta::ErasureInfo;
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
|
||||
use crate::disk::error_reduce::count_errs;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::store_api::{ObjectInfoOrErr, WalkOptions};
|
||||
use crate::{
|
||||
disk::{
|
||||
DiskAPI, DiskInfo, DiskOption, DiskStore,
|
||||
@@ -26,8 +25,8 @@ use crate::{
|
||||
endpoints::{Endpoints, PoolEndpoints},
|
||||
error::StorageError,
|
||||
global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure},
|
||||
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
|
||||
set_disk::SetDisks,
|
||||
store_api::{GetObjectReader, ListObjectVersionsInfo, ListObjectsV2Info, ObjectInfo, ObjectOptions, PutObjReader},
|
||||
store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
|
||||
};
|
||||
use futures::{
|
||||
@@ -47,8 +46,10 @@ use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
|
||||
use rustfs_storage_api::CompletePart;
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
use rustfs_storage_api::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DeletedObject, ListMultipartsInfo, ListPartsInfo,
|
||||
MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
|
||||
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListPartsInfo,
|
||||
MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectToDelete, PartInfo,
|
||||
WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
|
||||
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
|
||||
@@ -61,6 +62,11 @@ use tracing::warn;
|
||||
use tracing::{error, info};
|
||||
use uuid::Uuid;
|
||||
|
||||
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Sets {
|
||||
pub id: Uuid,
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
use crate::error::Error;
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_storage_api::{
|
||||
DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectToDelete,
|
||||
WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
use std::fmt::Debug;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
pub(crate) trait EcstoreObjectIO:
|
||||
rustfs_storage_api::ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = http::HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
+ 'static
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> EcstoreObjectIO for T where
|
||||
T: rustfs_storage_api::ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = http::HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
+ 'static
|
||||
{
|
||||
}
|
||||
|
||||
pub(crate) trait EcstoreObjectOperations:
|
||||
rustfs_storage_api::ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> EcstoreObjectOperations for T where
|
||||
T: rustfs_storage_api::ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
{
|
||||
}
|
||||
|
||||
pub(crate) trait EcstoreListOperations:
|
||||
rustfs_storage_api::ListOperations<
|
||||
Error = Error,
|
||||
ListObjectsV2Info = ListObjectsV2Info,
|
||||
ListObjectVersionsInfo = ListObjectVersionsInfo,
|
||||
ObjectInfoOrErr = ObjectInfoOrErr,
|
||||
WalkOptions = WalkOptions,
|
||||
WalkCancellation = CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> EcstoreListOperations for T where
|
||||
T: rustfs_storage_api::ListOperations<
|
||||
Error = Error,
|
||||
ListObjectsV2Info = ListObjectsV2Info,
|
||||
ListObjectVersionsInfo = ListObjectVersionsInfo,
|
||||
ObjectInfoOrErr = ObjectInfoOrErr,
|
||||
WalkOptions = WalkOptions,
|
||||
WalkCancellation = CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
{
|
||||
}
|
||||
@@ -53,16 +53,15 @@ use crate::notification_sys::get_global_notification_sys;
|
||||
use crate::pools::PoolMeta;
|
||||
use crate::rebalance::RebalanceMeta;
|
||||
use crate::rpc::RemoteClient;
|
||||
use crate::store_api::{ListObjectVersionsInfo, ObjectInfoOrErr, WalkOptions};
|
||||
use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
|
||||
use crate::tier::tier::TierConfigMgr;
|
||||
use crate::{
|
||||
bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
|
||||
disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk},
|
||||
endpoints::EndpointServerPools,
|
||||
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
|
||||
rpc::S3PeerSys,
|
||||
sets::Sets,
|
||||
store_api::{GetObjectReader, ListObjectsV2Info, ObjectInfo, ObjectOptions, PutObjReader},
|
||||
store_init,
|
||||
};
|
||||
use futures::future::join_all;
|
||||
@@ -75,11 +74,14 @@ use rustfs_config::server_config::{Config, get_global_server_config, set_global_
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_storage_api::HTTPRangeSpec;
|
||||
use rustfs_storage_api::{
|
||||
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
|
||||
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
|
||||
};
|
||||
use rustfs_storage_api::{
|
||||
HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf};
|
||||
use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
|
||||
use std::cmp::Ordering;
|
||||
@@ -100,6 +102,11 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
/// Check if a directory contains any xl.meta files (indicating actual S3 objects)
|
||||
/// This is used to determine if a bucket is empty for deletion purposes.
|
||||
async fn has_xlmeta_files(path: &std::path::Path) -> bool {
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
use super::*;
|
||||
use rustfs_storage_api::{
|
||||
CompletePart, DeletedObject, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, ObjectToDelete,
|
||||
PartInfo,
|
||||
};
|
||||
|
||||
pub trait ObjectIO:
|
||||
rustfs_storage_api::ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
+ 'static
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> ObjectIO for T where
|
||||
T: rustfs_storage_api::ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
+ 'static
|
||||
{
|
||||
}
|
||||
|
||||
/// Object-level storage operations (beyond basic I/O in ObjectIO).
|
||||
pub trait ObjectOperations:
|
||||
rustfs_storage_api::ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> ObjectOperations for T where
|
||||
T: rustfs_storage_api::ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
{
|
||||
}
|
||||
|
||||
/// Listing and walking operations.
|
||||
pub trait ListOperations:
|
||||
rustfs_storage_api::ListOperations<
|
||||
Error = Error,
|
||||
ListObjectsV2Info = ListObjectsV2Info,
|
||||
ListObjectVersionsInfo = ListObjectVersionsInfo,
|
||||
ObjectInfoOrErr = ObjectInfoOrErr,
|
||||
WalkOptions = WalkOptions,
|
||||
WalkCancellation = CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> ListOperations for T where
|
||||
T: rustfs_storage_api::ListOperations<
|
||||
Error = Error,
|
||||
ListObjectsV2Info = ListObjectsV2Info,
|
||||
ListObjectVersionsInfo = ListObjectVersionsInfo,
|
||||
ObjectInfoOrErr = ObjectInfoOrErr,
|
||||
WalkOptions = WalkOptions,
|
||||
WalkCancellation = CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
{
|
||||
}
|
||||
|
||||
/// Multipart upload operations.
|
||||
pub trait MultipartOperations:
|
||||
rustfs_storage_api::MultipartOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
PutObjectReader = PutObjReader,
|
||||
CompletePart = CompletePart,
|
||||
ListMultipartsInfo = ListMultipartsInfo,
|
||||
MultipartUploadResult = MultipartUploadResult,
|
||||
PartInfo = PartInfo,
|
||||
MultipartInfo = MultipartInfo,
|
||||
ListPartsInfo = ListPartsInfo,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> MultipartOperations for T where
|
||||
T: rustfs_storage_api::MultipartOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
PutObjectReader = PutObjReader,
|
||||
CompletePart = CompletePart,
|
||||
ListMultipartsInfo = ListMultipartsInfo,
|
||||
MultipartUploadResult = MultipartUploadResult,
|
||||
PartInfo = PartInfo,
|
||||
MultipartInfo = MultipartInfo,
|
||||
ListPartsInfo = ListPartsInfo,
|
||||
> + Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
{
|
||||
}
|
||||
|
||||
/// Healing and repair operations.
|
||||
pub trait HealOperations:
|
||||
rustfs_storage_api::HealOperations<Error = Error, HealResultItem = HealResultItem, HealOptions = HealOpts> + Send + Sync + Debug
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> HealOperations for T where
|
||||
T: rustfs_storage_api::HealOperations<Error = Error, HealResultItem = HealResultItem, HealOptions = HealOpts>
|
||||
+ Send
|
||||
+ Sync
|
||||
+ Debug
|
||||
{
|
||||
}
|
||||
|
||||
/// Namespace lock operations needed by consumers that only coordinate object
|
||||
/// mutations but do not require the full storage API surface.
|
||||
pub trait NamespaceLocking:
|
||||
rustfs_storage_api::NamespaceLocking<Error = Error, NamespaceLock = NamespaceLockWrapper> + Send + Sync + Debug + 'static
|
||||
{
|
||||
}
|
||||
|
||||
impl<T> NamespaceLocking for T where
|
||||
T: rustfs_storage_api::NamespaceLocking<Error = Error, NamespaceLock = NamespaceLockWrapper> + Send + Sync + Debug + 'static
|
||||
{
|
||||
}
|
||||
@@ -21,17 +21,21 @@ use crate::disk::{DiskInfo, DiskStore};
|
||||
use crate::error::{
|
||||
Error, Result, StorageError, is_all_not_found, is_all_volume_not_found, is_err_bucket_not_found, to_object_err,
|
||||
};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::store_api::{ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectInfoOrErr, ObjectOptions, WalkOptions};
|
||||
use crate::store::ECStore;
|
||||
use crate::store_utils::is_reserved_or_invalid_bucket;
|
||||
use crate::{store::ECStore, store_api::ListObjectsV2Info};
|
||||
use futures::future::join_all;
|
||||
use rand::seq::SliceRandom;
|
||||
use rustfs_filemeta::{
|
||||
MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
|
||||
merge_file_meta_versions,
|
||||
};
|
||||
use rustfs_storage_api::{ObjectOperations as _, VersionMarker, WalkVersionsSortOrder};
|
||||
use rustfs_storage_api::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo as StorageListObjectsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations as _,
|
||||
VersionMarker, WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
|
||||
};
|
||||
use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
@@ -48,6 +52,12 @@ const MAX_OBJECT_LIST: i32 = 1000;
|
||||
|
||||
const METACACHE_SHARE_PREFIX: bool = false;
|
||||
|
||||
type ListObjectsInfo = StorageListObjectsInfo<ObjectInfo>;
|
||||
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
|
||||
|
||||
fn normalize_max_keys(max_keys: i32) -> i32 {
|
||||
max_keys.min(MAX_OBJECT_LIST)
|
||||
}
|
||||
|
||||
@@ -49,11 +49,13 @@ use crate::{
|
||||
config::com::{CONFIG_PREFIX, read_config},
|
||||
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
|
||||
global::is_first_cluster_node_local,
|
||||
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
|
||||
storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations},
|
||||
store::ECStore,
|
||||
store_api::{ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
|
||||
};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
|
||||
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete};
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
|
||||
use s3s::S3ErrorCode;
|
||||
|
||||
@@ -1032,19 +1034,36 @@ impl TierConfigMgr {
|
||||
self.save_tiering_config(api).await
|
||||
}
|
||||
|
||||
pub async fn save_tiering_config<S: ObjectIO>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
|
||||
pub async fn save_tiering_config<S>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
let data = encode_external_tiering_config_blob(self)?;
|
||||
let config_file = tier_config_path(TIER_CONFIG_FILE);
|
||||
|
||||
self.save_config(api, &config_file, data).await
|
||||
}
|
||||
|
||||
pub async fn save_config<S: ObjectIO>(
|
||||
&self,
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
data: Bytes,
|
||||
) -> std::result::Result<(), std::io::Error> {
|
||||
pub async fn save_config<S>(&self, api: Arc<S>, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
self.save_config_with_opts(
|
||||
api,
|
||||
file,
|
||||
@@ -1057,13 +1076,24 @@ impl TierConfigMgr {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn save_config_with_opts<S: ObjectIO>(
|
||||
pub async fn save_config_with_opts<S>(
|
||||
&self,
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
data: Bytes,
|
||||
opts: &ObjectOptions,
|
||||
) -> std::result::Result<(), std::io::Error> {
|
||||
) -> std::result::Result<(), std::io::Error>
|
||||
where
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
debug!("save tier config:{}", file);
|
||||
let mut put_data = PutObjReader::from_vec(data.to_vec());
|
||||
let _ = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await?;
|
||||
@@ -1099,7 +1129,10 @@ impl TierConfigMgr {
|
||||
}
|
||||
}
|
||||
|
||||
async fn new_and_save_tiering_config<S: ObjectIO>(api: Arc<S>) -> Result<TierConfigMgr> {
|
||||
async fn new_and_save_tiering_config<S>(api: Arc<S>) -> Result<TierConfigMgr>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let mut cfg = TierConfigMgr {
|
||||
driver_cache: HashMap::new(),
|
||||
tiers: HashMap::new(),
|
||||
@@ -1158,12 +1191,15 @@ async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMg
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_tier_config_from_bucket<S: ObjectIO>(
|
||||
async fn read_tier_config_from_bucket<S>(
|
||||
api: Arc<S>,
|
||||
bucket: &str,
|
||||
path: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> io::Result<Option<Vec<u8>>> {
|
||||
) -> io::Result<Option<Vec<u8>>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let mut rd = match api.get_object_reader(bucket, path, None, HeaderMap::new(), opts).await {
|
||||
Ok(v) => v,
|
||||
Err(err) if is_err_config_not_found(&err) => return Ok(None),
|
||||
@@ -1176,7 +1212,10 @@ async fn read_tier_config_from_bucket<S: ObjectIO>(
|
||||
Ok(Some(data))
|
||||
}
|
||||
|
||||
async fn write_tier_config_to_rustfs<S: ObjectIO>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()> {
|
||||
async fn write_tier_config_to_rustfs<S>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let mut put_data = PutObjReader::from_vec(data.to_vec());
|
||||
api.put_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
@@ -1194,7 +1233,22 @@ async fn write_tier_config_to_rustfs<S: ObjectIO>(api: Arc<S>, path: &str, data:
|
||||
|
||||
pub async fn try_migrate_tiering_config<S>(api: Arc<S>)
|
||||
where
|
||||
S: ObjectIO + ObjectOperations,
|
||||
S: ObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
> + ObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
let target_path = tier_config_path(TIER_CONFIG_FILE);
|
||||
if api
|
||||
|
||||
@@ -16,12 +16,25 @@ use rustfs_common::heal_channel::HealOpts;
|
||||
use rustfs_ecstore::{
|
||||
disk::DiskStore,
|
||||
error::Error,
|
||||
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
|
||||
store::ECStore,
|
||||
store_api::{HealOperations, NamespaceLocking},
|
||||
};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_lock::NamespaceLockWrapper;
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_storage_api::{HealOperations as StorageHealOperations, NamespaceLocking as StorageNamespaceLocking, StorageAdminApi};
|
||||
use rustfs_storage_api::{
|
||||
CompletePart, DeletedObject, HTTPRangeSpec, HealOperations as StorageHealOperations, ListMultipartsInfo,
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListPartsInfo,
|
||||
MultipartInfo, MultipartOperations as StorageMultipartOperations, MultipartUploadResult,
|
||||
NamespaceLocking as StorageNamespaceLocking, ObjectIO as StorageObjectIO, ObjectInfoOrErr as StorageObjectInfoOrErr,
|
||||
ObjectOperations as StorageObjectOperations, ObjectToDelete, PartInfo, StorageAdminApi, WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
fn storage_admin_api_type_name<T>() -> &'static str
|
||||
where
|
||||
@@ -35,20 +48,6 @@ where
|
||||
std::any::type_name::<T>()
|
||||
}
|
||||
|
||||
fn namespace_locking_type_name<T>() -> &'static str
|
||||
where
|
||||
T: NamespaceLocking,
|
||||
{
|
||||
std::any::type_name::<T>()
|
||||
}
|
||||
|
||||
fn heal_operations_type_name<T>() -> &'static str
|
||||
where
|
||||
T: HealOperations,
|
||||
{
|
||||
std::any::type_name::<T>()
|
||||
}
|
||||
|
||||
fn storage_namespace_locking_type_name<T>() -> &'static str
|
||||
where
|
||||
T: StorageNamespaceLocking<Error = Error, NamespaceLock = NamespaceLockWrapper>,
|
||||
@@ -56,6 +55,68 @@ where
|
||||
std::any::type_name::<T>()
|
||||
}
|
||||
|
||||
fn storage_object_io_type_name<T>() -> &'static str
|
||||
where
|
||||
T: StorageObjectIO<
|
||||
Error = Error,
|
||||
RangeSpec = HTTPRangeSpec,
|
||||
HeaderMap = http::HeaderMap,
|
||||
ObjectOptions = ObjectOptions,
|
||||
ObjectInfo = ObjectInfo,
|
||||
GetObjectReader = GetObjectReader,
|
||||
PutObjectReader = PutObjReader,
|
||||
>,
|
||||
{
|
||||
std::any::type_name::<T>()
|
||||
}
|
||||
|
||||
fn storage_object_operations_type_name<T>() -> &'static str
|
||||
where
|
||||
T: StorageObjectOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
FileInfo = FileInfo,
|
||||
ObjectToDelete = ObjectToDelete,
|
||||
DeletedObject = DeletedObject,
|
||||
>,
|
||||
{
|
||||
std::any::type_name::<T>()
|
||||
}
|
||||
|
||||
fn storage_list_operations_type_name<T>() -> &'static str
|
||||
where
|
||||
T: rustfs_storage_api::ListOperations<
|
||||
Error = Error,
|
||||
ListObjectsV2Info = ListObjectsV2Info,
|
||||
ListObjectVersionsInfo = ListObjectVersionsInfo,
|
||||
ObjectInfoOrErr = ObjectInfoOrErr,
|
||||
WalkOptions = WalkOptions,
|
||||
WalkCancellation = CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
|
||||
>,
|
||||
{
|
||||
std::any::type_name::<T>()
|
||||
}
|
||||
|
||||
fn storage_multipart_operations_type_name<T>() -> &'static str
|
||||
where
|
||||
T: StorageMultipartOperations<
|
||||
Error = Error,
|
||||
ObjectInfo = ObjectInfo,
|
||||
ObjectOptions = ObjectOptions,
|
||||
PutObjectReader = PutObjReader,
|
||||
CompletePart = CompletePart,
|
||||
ListMultipartsInfo = ListMultipartsInfo,
|
||||
MultipartUploadResult = MultipartUploadResult,
|
||||
PartInfo = PartInfo,
|
||||
MultipartInfo = MultipartInfo,
|
||||
ListPartsInfo = ListPartsInfo,
|
||||
>,
|
||||
{
|
||||
std::any::type_name::<T>()
|
||||
}
|
||||
|
||||
fn storage_heal_operations_type_name<T>() -> &'static str
|
||||
where
|
||||
T: StorageHealOperations<Error = Error, HealResultItem = HealResultItem, HealOptions = HealOpts>,
|
||||
@@ -68,21 +129,31 @@ fn ecstore_implements_storage_admin_api_contract() {
|
||||
assert!(storage_admin_api_type_name::<ECStore>().ends_with("::ECStore"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ecstore_implements_namespace_locking_contract() {
|
||||
assert!(namespace_locking_type_name::<ECStore>().ends_with("::ECStore"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ecstore_implements_heal_operations_contract() {
|
||||
assert!(heal_operations_type_name::<ECStore>().ends_with("::ECStore"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ecstore_implements_storage_namespace_locking_contract() {
|
||||
assert!(storage_namespace_locking_type_name::<ECStore>().ends_with("::ECStore"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ecstore_implements_storage_object_io_contract() {
|
||||
assert!(storage_object_io_type_name::<ECStore>().ends_with("::ECStore"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ecstore_implements_storage_object_operations_contract() {
|
||||
assert!(storage_object_operations_type_name::<ECStore>().ends_with("::ECStore"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ecstore_implements_storage_list_operations_contract() {
|
||||
assert!(storage_list_operations_type_name::<ECStore>().ends_with("::ECStore"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ecstore_implements_storage_multipart_operations_contract() {
|
||||
assert!(storage_multipart_operations_type_name::<ECStore>().ends_with("::ECStore"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ecstore_implements_storage_heal_operations_contract() {
|
||||
assert!(storage_heal_operations_type_name::<ECStore>().ends_with("::ECStore"));
|
||||
|
||||
@@ -8,7 +8,7 @@ use rustfs_ecstore::bitrot::create_bitrot_reader;
|
||||
use rustfs_ecstore::disk::endpoint::Endpoint;
|
||||
use rustfs_ecstore::disk::{DiskAPI as _, DiskOption, new_disk};
|
||||
use rustfs_ecstore::erasure_coding::Erasure;
|
||||
use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use rustfs_ecstore::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
@@ -29,6 +29,6 @@ pub(crate) use rustfs_ecstore::global::GLOBAL_LOCAL_DISK_MAP;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::disk::{DiskOption, new_disk};
|
||||
|
||||
pub type HealObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||
pub type HealObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
|
||||
pub type HealPutObjReader = rustfs_ecstore::store_api::PutObjReader;
|
||||
pub type HealObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
|
||||
pub type HealObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
|
||||
pub type HealPutObjReader = rustfs_ecstore::object_api::PutObjReader;
|
||||
|
||||
@@ -17,8 +17,8 @@ use std::sync::Arc;
|
||||
pub(crate) const IAM_CONFIG_ROOT_PREFIX: &str = rustfs_ecstore::config::RUSTFS_CONFIG_PREFIX;
|
||||
|
||||
pub(crate) type IamEcstoreError = rustfs_ecstore::error::Error;
|
||||
pub(crate) type IamConfigObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||
pub(crate) type IamConfigObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
|
||||
pub(crate) type IamConfigObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
|
||||
pub(crate) type IamConfigObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
|
||||
pub(crate) type IamStorageError = rustfs_ecstore::error::StorageError;
|
||||
pub(crate) type IamStorageResult<T> = rustfs_ecstore::error::Result<T>;
|
||||
pub(crate) type IamStore = rustfs_ecstore::store::ECStore;
|
||||
|
||||
@@ -12,5 +12,5 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(super) type IamObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||
pub(super) type IamObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
|
||||
pub(super) type IamObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
|
||||
pub(super) type IamObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
|
||||
|
||||
@@ -18,7 +18,7 @@ use rustfs_ecstore::{config, global};
|
||||
|
||||
use crate::event::NotifyObjectInfo;
|
||||
|
||||
type EcstoreObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||
type EcstoreObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum NotifyConfigStoreError {
|
||||
|
||||
@@ -18,10 +18,10 @@ pub(super) type SwiftBucketMetadata = rustfs_ecstore::bucket::metadata::BucketMe
|
||||
pub(super) type SwiftStorageResult<T> = rustfs_ecstore::error::Result<T>;
|
||||
pub(super) type SwiftStore = rustfs_ecstore::store::ECStore;
|
||||
|
||||
pub type SwiftGetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
|
||||
pub type SwiftObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||
pub type SwiftObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
|
||||
pub type SwiftPutObjReader = rustfs_ecstore::store_api::PutObjReader;
|
||||
pub type SwiftGetObjectReader = rustfs_ecstore::object_api::GetObjectReader;
|
||||
pub type SwiftObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
|
||||
pub type SwiftObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
|
||||
pub type SwiftPutObjReader = rustfs_ecstore::object_api::PutObjReader;
|
||||
|
||||
pub fn resolve_swift_object_store_handle() -> Option<Arc<SwiftStore>> {
|
||||
rustfs_ecstore::resolve_object_store_handle()
|
||||
|
||||
@@ -12,9 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) type SelectGetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
|
||||
pub(crate) type SelectObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||
pub(crate) type SelectObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
|
||||
pub(crate) type SelectGetObjectReader = rustfs_ecstore::object_api::GetObjectReader;
|
||||
pub(crate) type SelectObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
|
||||
pub(crate) type SelectObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
|
||||
pub(crate) type SelectStorageError = rustfs_ecstore::error::StorageError;
|
||||
pub(crate) type SelectStore = rustfs_ecstore::store::ECStore;
|
||||
|
||||
|
||||
@@ -58,11 +58,11 @@ pub(crate) use rustfs_ecstore::config::init as init_ecstore_config_for_scanner_t
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::disk::{DiskOption, endpoint::Endpoint, new_disk};
|
||||
|
||||
pub type ScannerGetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
|
||||
pub type ScannerObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||
pub type ScannerObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
|
||||
pub type ScannerGetObjectReader = rustfs_ecstore::object_api::GetObjectReader;
|
||||
pub type ScannerObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
|
||||
pub type ScannerObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
|
||||
pub type ScannerObjectToDelete = ObjectToDelete;
|
||||
pub type ScannerPutObjReader = rustfs_ecstore::store_api::PutObjReader;
|
||||
pub type ScannerPutObjReader = rustfs_ecstore::object_api::PutObjReader;
|
||||
|
||||
pub(crate) fn resolve_scanner_object_store_handle() -> Option<Arc<ECStore>> {
|
||||
rustfs_ecstore::resolve_object_store_handle()
|
||||
|
||||
@@ -5,19 +5,17 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
## Current Context
|
||||
|
||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||
- Branch: `overtrue/arch-foreground-scanner-admission-snapshots`
|
||||
- Baseline: `origin/main` after `rustfs/rustfs#3606`
|
||||
(`7cb7aefc3bfd33cfaa46c718db15f00b0471fe88`).
|
||||
- Branch: `overtrue/arch-ecstore-object-api-alias-prune`
|
||||
- Baseline: stacked after `rustfs/rustfs#3614`
|
||||
(`3740e65246aed8c8ee9685ed0234997c7d54e1b5`).
|
||||
- PR type for this branch: `consumer-migration`
|
||||
- Runtime behavior changes: none.
|
||||
- Rust code changes: extend the RustFS workload admission registry from
|
||||
repair/replication snapshots to foreground-read, scanner, and metadata owner
|
||||
snapshots.
|
||||
- CI/script changes: extend migration guard coverage for RustFS workload
|
||||
admission provider implementations.
|
||||
- Docs changes: add RustFS runtime owner provider notes to
|
||||
[`workload-admission-contracts.md`](workload-admission-contracts.md) and
|
||||
record the API-059/R-019 provider slice.
|
||||
- Rust code changes: remove unused public `rustfs-storage-api` passthrough
|
||||
aliases from ECStore `object_api` and bind the ECStore contract test directly
|
||||
to the storage-api generic list/delete/walk contracts.
|
||||
- CI/script changes: make the migration guard reject restoring ECStore
|
||||
`object_api` storage-api passthrough aliases.
|
||||
- Docs changes: record the API-066 ECStore object API alias prune slice.
|
||||
|
||||
## Phase 0 Tasks
|
||||
|
||||
@@ -214,6 +212,112 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
check, migration and layer guards, formatting, diff hygiene, risk scan, and
|
||||
three-expert review.
|
||||
|
||||
- [x] `API-060` Remove heal and namespace-lock operation compatibility facades.
|
||||
- Completed slice: remove the old ECStore `store_api::HealOperations` and
|
||||
`store_api::NamespaceLocking` compatibility subtraits after ECStore storage
|
||||
types already implemented the shared `rustfs_storage_api` contracts
|
||||
directly.
|
||||
- Acceptance: internal ECStore bounds and compile-time coverage use the
|
||||
shared storage-api heal and namespace-lock contracts directly, while the
|
||||
remaining object/list/multipart compatibility bindings stay unchanged for
|
||||
their active internal consumers.
|
||||
- Must preserve: heal operation behavior, namespace-lock acquisition,
|
||||
replication resync locking, rebalance metadata locking, object I/O,
|
||||
multipart, list, and storage hot paths.
|
||||
- Verification: focused ECStore contract tests, focused ECStore library
|
||||
check, migration and layer guards, formatting, diff hygiene, risk scan, and
|
||||
three-expert review.
|
||||
|
||||
- [x] `API-061` Remove public ECStore object operation compatibility facades.
|
||||
- Completed slice: remove the old public ECStore `store_api` object, list,
|
||||
and multipart operation compatibility subtraits, and keep internal generic
|
||||
bounds on crate-private storage-api contract constraints instead of public
|
||||
downstream compatibility traits.
|
||||
- Acceptance: `store_api` no longer exports public operation compatibility
|
||||
traits, ECStore direct storage-api compile-time coverage includes object,
|
||||
object-operation, list, multipart, namespace-lock, heal, and admin
|
||||
contracts, and remaining public `store_api` exports are DTO/reader
|
||||
compatibility paths only.
|
||||
- Must preserve: object I/O, list/walk behavior, multipart behavior, config
|
||||
persistence, tier config migration, rebalance metadata locking, lifecycle
|
||||
journal handling, replication MRF/resync persistence, and downstream DTO
|
||||
import compatibility.
|
||||
- Verification: focused ECStore contract tests, focused ECStore library
|
||||
check, migration and layer guards, formatting, diff hygiene, risk scan, and
|
||||
three-expert review.
|
||||
|
||||
- [x] `API-062` Establish explicit ECStore object API boundary.
|
||||
- Completed slice: add `rustfs_ecstore::object_api` as the explicit public
|
||||
path for ECStore-owned object DTO and reader contracts, then migrate
|
||||
RustFS, scanner, heal, IAM, Swift, S3 Select, notify, and ECStore
|
||||
integration-test compatibility aliases away from the legacy public
|
||||
`store_api` path.
|
||||
- Acceptance: external compatibility boundary modules no longer reference
|
||||
`rustfs_ecstore::store_api` for ECStore-owned object DTO and reader
|
||||
aliases, while `store_api` remains available only as the old internal
|
||||
implementation module pending final compatibility removal.
|
||||
- Must preserve: object metadata shape, option defaults, reader/writer
|
||||
behavior, Swift/scanner/heal/IAM/S3 Select/notify boundary semantics, and
|
||||
all storage hot paths.
|
||||
- Verification: focused ECStore/RustFS/downstream compile checks, migration
|
||||
guard, formatting, diff hygiene, risk scan, and three-expert review.
|
||||
|
||||
- [x] `API-063` Make legacy ECStore store API module private.
|
||||
- Completed slice: remove `rustfs_ecstore::store_api` from the public crate
|
||||
module surface after external compatibility boundaries moved to
|
||||
`rustfs_ecstore::object_api`.
|
||||
- Acceptance: ECStore object DTO and reader compatibility remains available
|
||||
through `object_api`, integration contract tests consume the new public
|
||||
path, and migration rules reject restoring `pub mod store_api`.
|
||||
- Must preserve: internal ECStore object DTO definitions, reader/writer
|
||||
behavior, storage-api trait bindings, and downstream object/list/multipart
|
||||
compile-time contracts.
|
||||
- Verification: focused ECStore contract tests, migration guard, formatting,
|
||||
diff hygiene, risk scan, and three-expert review.
|
||||
|
||||
- [x] `API-064` Retire the ECStore store API module name.
|
||||
- Completed slice: move ECStore object DTO, reader, and option definitions
|
||||
from the private `store_api` module into the public `object_api` module,
|
||||
then migrate ECStore internal imports to `crate::object_api`.
|
||||
- Acceptance: no ECStore `store_api` module file or directory remains, public
|
||||
consumers keep using `rustfs_ecstore::object_api`, and migration rules
|
||||
reject restoring the retired module path.
|
||||
- Must preserve: object metadata shape, reader/writer behavior, storage-api
|
||||
contract bindings, object/list/multipart behavior, and downstream public
|
||||
object API compatibility.
|
||||
- Verification: focused ECStore compile checks, migration guard, formatting,
|
||||
diff hygiene, risk scan, and three-expert review.
|
||||
|
||||
- [x] `API-065` Use storage-api list contracts inside ECStore.
|
||||
- Completed slice: migrate ECStore internal list response, walk options, and
|
||||
walk result bindings to local aliases over the generic `rustfs-storage-api`
|
||||
contracts, including replication worker trait bounds, while retaining
|
||||
public `rustfs_ecstore::object_api` aliases for downstream compatibility.
|
||||
- Acceptance: ECStore implementation modules no longer import list/walk
|
||||
compatibility aliases from `crate::object_api`, and migration rules reject
|
||||
reintroducing those internal imports.
|
||||
- Must preserve: list response shape, walk result item shape, object metadata
|
||||
shape, storage-api trait bindings, and downstream public object API
|
||||
compatibility.
|
||||
- Verification: focused ECStore compile checks, migration guard, formatting,
|
||||
diff hygiene, risk scan, and three-expert review.
|
||||
|
||||
- [x] `API-066` Prune ECStore object API storage aliases.
|
||||
- Completed slice: remove unused public storage-api passthrough aliases from
|
||||
ECStore `object_api` for list responses, walk options, walk result items,
|
||||
and delete-object DTOs, then bind the ECStore contract test directly to the
|
||||
generic `rustfs-storage-api` contracts.
|
||||
- Acceptance: ECStore `object_api` no longer exposes storage-api passthrough
|
||||
aliases, the storage contract test still proves ECStore implements the
|
||||
storage-api traits with the same associated concrete types, and migration
|
||||
rules reject restoring the object_api passthrough aliases.
|
||||
- Must preserve: ECStore-owned object metadata, object options, reader/writer
|
||||
types, storage-api trait associated type bindings, list/delete/walk response
|
||||
shapes, and runtime behavior.
|
||||
- Verification: focused ECStore compile checks, storage contract test,
|
||||
downstream compile checks, migration and layer guards, formatting, diff
|
||||
hygiene, risk scan, full pre-commit, and three-expert review.
|
||||
|
||||
- [x] `TEST-PRTYPE-001` Check PR type enum consistency.
|
||||
- Acceptance: `./scripts/check_architecture_migration_rules.sh` parses the
|
||||
allowed PR types from [`crate-boundaries.md`](crate-boundaries.md) and fails
|
||||
|
||||
@@ -39,7 +39,7 @@ pub(crate) use rustfs_ecstore::rpc::{
|
||||
pub(crate) use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk_path, find_local_disk_by_ref};
|
||||
|
||||
pub(crate) type GetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
|
||||
pub(crate) type ObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||
pub(crate) type ObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
|
||||
pub(crate) type PutObjReader = rustfs_ecstore::store_api::PutObjReader;
|
||||
pub(crate) type GetObjectReader = rustfs_ecstore::object_api::GetObjectReader;
|
||||
pub(crate) type ObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
|
||||
pub(crate) type ObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
|
||||
pub(crate) type PutObjReader = rustfs_ecstore::object_api::PutObjReader;
|
||||
|
||||
@@ -57,6 +57,8 @@ STORE_API_OBJECT_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_object_helper_reexp
|
||||
STORE_API_RANGE_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_range_helper_reexports.txt"
|
||||
STORE_API_LIST_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_list_helper_reexports.txt"
|
||||
STORE_API_LIST_RESPONSE_REEXPORTS_FILE="${TMP_DIR}/store_api_list_response_reexports.txt"
|
||||
ECSTORE_OBJECT_API_LIST_ALIAS_INTERNAL_HITS_FILE="${TMP_DIR}/ecstore_object_api_list_alias_internal_hits.txt"
|
||||
ECSTORE_OBJECT_API_STORAGE_ALIAS_HITS_FILE="${TMP_DIR}/ecstore_object_api_storage_alias_hits.txt"
|
||||
STORE_API_DELETE_DTO_REEXPORTS_FILE="${TMP_DIR}/store_api_delete_dto_reexports.txt"
|
||||
STORE_API_DELETE_DTO_INTERNAL_HITS_FILE="${TMP_DIR}/store_api_delete_dto_internal_hits.txt"
|
||||
STORE_API_LIFECYCLE_HELPER_DEFINITION_HITS_FILE="${TMP_DIR}/store_api_lifecycle_helper_definition_hits.txt"
|
||||
@@ -73,6 +75,8 @@ PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE="${TMP_DIR}/production_unused_compat_al
|
||||
BROAD_STORE_API_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/broad_store_api_compat_reexport_hits.txt"
|
||||
NESTED_STORE_API_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/nested_store_api_compat_module_hits.txt"
|
||||
UNAPPROVED_STORE_API_COMPAT_ALIAS_HITS_FILE="${TMP_DIR}/unapproved_store_api_compat_alias_hits.txt"
|
||||
PUBLIC_STORE_API_MODULE_HITS_FILE="${TMP_DIR}/public_store_api_module_hits.txt"
|
||||
STORE_API_MODULE_PATH_HITS_FILE="${TMP_DIR}/store_api_module_path_hits.txt"
|
||||
ECSTORE_COMPAT_PASSTHROUGH_EXPECTED_FILE="${TMP_DIR}/ecstore_compat_passthrough_expected.txt"
|
||||
ECSTORE_COMPAT_PASSTHROUGH_ACTUAL_FILE="${TMP_DIR}/ecstore_compat_passthrough_actual.txt"
|
||||
ECSTORE_COMPAT_PASSTHROUGH_DIFF_FILE="${TMP_DIR}/ecstore_compat_passthrough_diff.txt"
|
||||
@@ -357,7 +361,7 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api::\{[^}]*\b(?:BucketInfo|BucketOptions|DeleteBucketOptions|MakeBucketOptions|SRBucketDeleteOp)\b' \
|
||||
crates/ecstore/src/store_api.rs || true
|
||||
crates/ecstore/src/store_api.rs 2>/dev/null || true
|
||||
) >"$STORE_API_BUCKET_DTO_REEXPORTS_FILE"
|
||||
|
||||
if [[ -s "$STORE_API_BUCKET_DTO_REEXPORTS_FILE" ]]; then
|
||||
@@ -366,8 +370,17 @@ fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api::\{[^}]*\bBucketOperations\b|pub trait BucketOperations\b' \
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/traits.rs || true
|
||||
store_api_bucket_operation_scan_targets=()
|
||||
if [[ -f crates/ecstore/src/store_api.rs ]]; then
|
||||
store_api_bucket_operation_scan_targets+=(crates/ecstore/src/store_api.rs)
|
||||
fi
|
||||
if [[ -f crates/ecstore/src/store_api/traits.rs ]]; then
|
||||
store_api_bucket_operation_scan_targets+=(crates/ecstore/src/store_api/traits.rs)
|
||||
fi
|
||||
if (( ${#store_api_bucket_operation_scan_targets[@]} > 0 )); then
|
||||
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api::\{[^}]*\bBucketOperations\b|pub trait BucketOperations\b' \
|
||||
"${store_api_bucket_operation_scan_targets[@]}" || true
|
||||
fi
|
||||
) >"$STORE_API_BUCKET_OPERATION_HITS_FILE"
|
||||
|
||||
if [[ -s "$STORE_API_BUCKET_OPERATION_HITS_FILE" ]]; then
|
||||
@@ -377,7 +390,7 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api::\{[^}]*\b(?:CompletePart|ListMultipartsInfo|ListPartsInfo|MultipartInfo|MultipartUploadResult|PartInfo)\b|pub struct (?:CompletePart|ListMultipartsInfo|ListPartsInfo|MultipartInfo|MultipartUploadResult|PartInfo)\b' \
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs 2>/dev/null || true
|
||||
) >"$STORE_API_MULTIPART_DTO_REEXPORTS_FILE"
|
||||
|
||||
if [[ -s "$STORE_API_MULTIPART_DTO_REEXPORTS_FILE" ]]; then
|
||||
@@ -387,7 +400,7 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:HTTPPreconditions|ObjectLockRetentionOptions|ObjectPreconditionError|ObjectPreconditionPart|ObjectPreconditionState)\b|::(?:HTTPPreconditions|ObjectLockRetentionOptions|ObjectPreconditionError|ObjectPreconditionPart|ObjectPreconditionState)\b)|pub (?:enum ObjectPreconditionError|struct (?:HTTPPreconditions|ObjectLockRetentionOptions|ObjectPreconditionPart|ObjectPreconditionState))\b' \
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs 2>/dev/null || true
|
||||
) >"$STORE_API_OBJECT_HELPER_REEXPORTS_FILE"
|
||||
|
||||
if [[ -s "$STORE_API_OBJECT_HELPER_REEXPORTS_FILE" ]]; then
|
||||
@@ -397,7 +410,7 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:HTTPRangeError|HTTPRangeSpec)\b|::(?:HTTPRangeError|HTTPRangeSpec)\b)|pub (?:enum HTTPRangeError|struct HTTPRangeSpec)\b' \
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs crates/ecstore/src/store_api/readers.rs || true
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs crates/ecstore/src/store_api/readers.rs 2>/dev/null || true
|
||||
) >"$STORE_API_RANGE_HELPER_REEXPORTS_FILE"
|
||||
|
||||
if [[ -s "$STORE_API_RANGE_HELPER_REEXPORTS_FILE" ]]; then
|
||||
@@ -407,7 +420,7 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:VersionMarker|WalkOptions|WalkVersionsSortOrder)\b|::(?:VersionMarker|WalkOptions|WalkVersionsSortOrder)\b)|pub (?:enum (?:VersionMarker|WalkVersionsSortOrder)|struct WalkOptions)\b' \
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs 2>/dev/null || true
|
||||
) >"$STORE_API_LIST_HELPER_REEXPORTS_FILE"
|
||||
|
||||
if [[ -s "$STORE_API_LIST_HELPER_REEXPORTS_FILE" ]]; then
|
||||
@@ -417,7 +430,7 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ListOperations|ObjectInfoOrErr)\b|::(?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ListOperations|ObjectInfoOrErr)\b)|pub struct (?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ObjectInfoOrErr)\b' \
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs 2>/dev/null || true
|
||||
) >"$STORE_API_LIST_RESPONSE_REEXPORTS_FILE"
|
||||
|
||||
if [[ -s "$STORE_API_LIST_RESPONSE_REEXPORTS_FILE" ]]; then
|
||||
@@ -427,7 +440,7 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:DeletedObject|ObjectToDelete)\b|::(?:DeletedObject|ObjectToDelete)\b)|pub struct (?:DeletedObject|ObjectToDelete)\b' \
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true
|
||||
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs 2>/dev/null || true
|
||||
) >"$STORE_API_DELETE_DTO_REEXPORTS_FILE"
|
||||
|
||||
if [[ -s "$STORE_API_DELETE_DTO_REEXPORTS_FILE" ]]; then
|
||||
@@ -484,6 +497,34 @@ if [[ -s "$STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE" ]]; then
|
||||
report_failure "external list response consumers must use rustfs-storage-api contracts: $(paste -sd '; ' "$STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
find crates/ecstore/src -type f -name '*.rs' -print0 |
|
||||
xargs -0 perl -0ne '
|
||||
while (/crate::object_api(?:::\{[^}]*\b(?:ListObjectsInfo|ListObjectsV2Info|ListObjectVersionsInfo|ObjectInfoOrErr|WalkOptions)\b|::(?:ListObjectsInfo|ListObjectsV2Info|ListObjectVersionsInfo|ObjectInfoOrErr|WalkOptions)\b)/sg) {
|
||||
my $prefix = substr($_, 0, $-[0]);
|
||||
my $line = ($prefix =~ tr/\n//) + 1;
|
||||
my $match = $&;
|
||||
$match =~ s/\s+/ /g;
|
||||
print "$ARGV:$line:$match\n";
|
||||
}
|
||||
' || true
|
||||
) >"$ECSTORE_OBJECT_API_LIST_ALIAS_INTERNAL_HITS_FILE"
|
||||
|
||||
if [[ -s "$ECSTORE_OBJECT_API_LIST_ALIAS_INTERNAL_HITS_FILE" ]]; then
|
||||
report_failure "ECStore internal list/walk consumers must bind rustfs-storage-api generic contracts directly: $(paste -sd '; ' "$ECSTORE_OBJECT_API_LIST_ALIAS_INTERNAL_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'pub type (?:ListObjectsInfo|ListObjectsV2Info|ListObjectVersionsInfo|ObjectInfoOrErr|WalkOptions|ObjectToDelete|DeletedObject)\b' \
|
||||
crates/ecstore/src/object_api/types.rs || true
|
||||
) >"$ECSTORE_OBJECT_API_STORAGE_ALIAS_HITS_FILE"
|
||||
|
||||
if [[ -s "$ECSTORE_OBJECT_API_STORAGE_ALIAS_HITS_FILE" ]]; then
|
||||
report_failure "ECStore object_api must not re-export storage-api passthrough aliases: $(paste -sd '; ' "$ECSTORE_OBJECT_API_STORAGE_ALIAS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'rustfs_ecstore::store_api(?:::\{[^}]*\b(?:ObjectIO|ObjectOperations|ListOperations|MultipartOperations|HealOperations|NamespaceLocking)\b|::(?:ObjectIO|ObjectOperations|ListOperations|MultipartOperations|HealOperations|NamespaceLocking)\b)' \
|
||||
@@ -613,14 +654,27 @@ fi
|
||||
--glob '*storage_compat.rs' \
|
||||
--glob '!crates/ecstore/**' \
|
||||
--glob '!crates/e2e_test/**' || true
|
||||
) | awk '
|
||||
$0 !~ /^[^:]+:[0-9]+:[[:space:]]*(pub(\([^)]*\))?[[:space:]]+)?type[[:space:]]+[A-Za-z0-9_]+[[:space:]]*=[[:space:]]*rustfs_ecstore::store_api::(GetObjectReader|ObjectInfo|ObjectOptions|PutObjReader);$/ {
|
||||
print
|
||||
}
|
||||
' >"$UNAPPROVED_STORE_API_COMPAT_ALIAS_HITS_FILE"
|
||||
) >"$UNAPPROVED_STORE_API_COMPAT_ALIAS_HITS_FILE"
|
||||
|
||||
if [[ -s "$UNAPPROVED_STORE_API_COMPAT_ALIAS_HITS_FILE" ]]; then
|
||||
report_failure "storage compatibility boundaries may only keep explicit ECStore-owned store_api type aliases: $(paste -sd '; ' "$UNAPPROVED_STORE_API_COMPAT_ALIAS_HITS_FILE")"
|
||||
report_failure "storage compatibility boundaries must use rustfs_ecstore::object_api for ECStore-owned object DTO and reader aliases: $(paste -sd '; ' "$UNAPPROVED_STORE_API_COMPAT_ALIAS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
if rg -n --no-heading '^\s*pub\s+mod\s+store_api\s*;' "$ROOT_DIR/crates/ecstore/src/lib.rs" >"$PUBLIC_STORE_API_MODULE_HITS_FILE"; then
|
||||
report_failure "ECStore store_api must remain private; expose ECStore-owned object DTO and reader aliases through rustfs_ecstore::object_api"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
{
|
||||
[[ -e crates/ecstore/src/store_api.rs ]] && printf '%s\n' 'crates/ecstore/src/store_api.rs'
|
||||
[[ -d crates/ecstore/src/store_api ]] && printf '%s\n' 'crates/ecstore/src/store_api'
|
||||
true
|
||||
}
|
||||
) >"$STORE_API_MODULE_PATH_HITS_FILE"
|
||||
|
||||
if [[ -s "$STORE_API_MODULE_PATH_HITS_FILE" ]]; then
|
||||
report_failure "legacy ECStore store_api module files must not be restored: $(paste -sd '; ' "$STORE_API_MODULE_PATH_HITS_FILE")"
|
||||
fi
|
||||
|
||||
cat >"$ECSTORE_COMPAT_PASSTHROUGH_EXPECTED_FILE" <<'EOF'
|
||||
@@ -652,50 +706,40 @@ fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading 'async fn (get_object_reader|put_object|get_object_info|verify_object_integrity|copy_object|delete_object_version|delete_object|delete_objects|put_object_metadata|get_object_tags|put_object_tags|delete_object_tags|add_partial|transition_object|restore_transitioned_object|list_multipart_uploads|new_multipart_upload|copy_object_part|put_object_part|get_multipart_info|list_object_parts|abort_multipart_upload|complete_multipart_upload|heal_format|heal_bucket|heal_object|get_pool_and_set|check_abandoned_parts|new_ns_lock)\b' \
|
||||
crates/ecstore/src/store_api/traits.rs || true
|
||||
if [[ -f crates/ecstore/src/store_api/traits.rs ]]; then
|
||||
rg -n --no-heading 'async fn (get_object_reader|put_object|get_object_info|verify_object_integrity|copy_object|delete_object_version|delete_object|delete_objects|put_object_metadata|get_object_tags|put_object_tags|delete_object_tags|add_partial|transition_object|restore_transitioned_object|list_multipart_uploads|new_multipart_upload|copy_object_part|put_object_part|get_multipart_info|list_object_parts|abort_multipart_upload|complete_multipart_upload|heal_format|heal_bucket|heal_object|get_pool_and_set|check_abandoned_parts|new_ns_lock)\b' \
|
||||
crates/ecstore/src/store_api/traits.rs || true
|
||||
fi
|
||||
) >"$STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE"
|
||||
|
||||
if [[ -s "$STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE" ]]; then
|
||||
report_failure "old ecstore operation method signatures reintroduced: $(paste -sd '; ' "$STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE")"
|
||||
fi
|
||||
|
||||
require_source_contains \
|
||||
"crates/ecstore/src/store_api/traits.rs" \
|
||||
"rustfs_storage_api::ObjectIO<" \
|
||||
"ECStore ObjectIO compatibility binding"
|
||||
require_source_contains \
|
||||
"crates/ecstore/src/store_api/traits.rs" \
|
||||
"rustfs_storage_api::ObjectOperations<" \
|
||||
"ECStore ObjectOperations compatibility binding"
|
||||
require_source_contains \
|
||||
"crates/ecstore/src/store_api/traits.rs" \
|
||||
"rustfs_storage_api::MultipartOperations<" \
|
||||
"ECStore MultipartOperations compatibility binding"
|
||||
require_source_contains \
|
||||
"crates/ecstore/src/store_api/traits.rs" \
|
||||
"rustfs_storage_api::HealOperations<" \
|
||||
"ECStore HealOperations compatibility binding"
|
||||
require_source_contains \
|
||||
"crates/ecstore/src/store_api/traits.rs" \
|
||||
"rustfs_storage_api::NamespaceLocking<" \
|
||||
"ECStore NamespaceLocking compatibility binding"
|
||||
require_source_contains \
|
||||
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
|
||||
"fn ecstore_implements_storage_admin_api_contract()" \
|
||||
"ECStore StorageAdminApi compile-time coverage test"
|
||||
require_source_contains \
|
||||
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
|
||||
"fn ecstore_implements_namespace_locking_contract()" \
|
||||
"ECStore NamespaceLocking compile-time coverage test"
|
||||
require_source_contains \
|
||||
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
|
||||
"fn ecstore_implements_heal_operations_contract()" \
|
||||
"ECStore HealOperations compile-time coverage test"
|
||||
require_source_contains \
|
||||
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
|
||||
"fn ecstore_implements_storage_namespace_locking_contract()" \
|
||||
"ECStore storage-api NamespaceLocking compile-time coverage test"
|
||||
require_source_contains \
|
||||
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
|
||||
"fn ecstore_implements_storage_object_io_contract()" \
|
||||
"ECStore storage-api ObjectIO compile-time coverage test"
|
||||
require_source_contains \
|
||||
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
|
||||
"fn ecstore_implements_storage_object_operations_contract()" \
|
||||
"ECStore storage-api ObjectOperations compile-time coverage test"
|
||||
require_source_contains \
|
||||
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
|
||||
"fn ecstore_implements_storage_list_operations_contract()" \
|
||||
"ECStore storage-api ListOperations compile-time coverage test"
|
||||
require_source_contains \
|
||||
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
|
||||
"fn ecstore_implements_storage_multipart_operations_contract()" \
|
||||
"ECStore storage-api MultipartOperations compile-time coverage test"
|
||||
require_source_contains \
|
||||
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
|
||||
"fn ecstore_implements_storage_heal_operations_contract()" \
|
||||
|
||||
Reference in New Issue
Block a user