refactor: remove ecstore operation compat facades (#3608)

This commit is contained in:
安正超
2026-06-19 17:01:12 +08:00
committed by GitHub
parent 132c8ae77d
commit 54ffbedbc8
42 changed files with 925 additions and 393 deletions
@@ -35,9 +35,9 @@ use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_ver
use crate::event_notification::{EventArgs, send_event};
use crate::global::GLOBAL_LocalNodeName;
use crate::global::{GLOBAL_LifecycleSys, GLOBAL_TierConfigMgr, get_global_deployment_id};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::set_disk::{MAX_PARTS_COUNT, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, SetDisks};
use crate::store::ECStore;
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::tier::warm_backend::WarmBackendGetOpts;
use async_channel::{Receiver as A_Receiver, Sender as A_Sender, bounded};
use futures::Future;
@@ -2863,9 +2863,9 @@ mod tests {
use crate::disk::endpoint::Endpoint;
use crate::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::error::is_err_invalid_upload_id;
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
use crate::set_disk::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
use crate::store::ECStore;
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader};
use futures::FutureExt;
use rustfs_common::metrics::{IlmAction, global_metrics};
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
+1 -1
View File
@@ -26,7 +26,7 @@ use time::{self, Duration, OffsetDateTime};
use tracing::debug;
use uuid::Uuid;
use crate::store_api::ObjectInfo;
use crate::object_api::ObjectInfo;
pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending";
@@ -23,9 +23,10 @@ use crate::bucket::lifecycle::tier_sweeper::{Jentry, delete_object_from_remote_t
use crate::config::com::{delete_config, read_config, save_config};
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::store::ECStore;
use crate::store_api::{ObjectIO, ObjectOperations};
use rustfs_storage_api::ListOperations as _;
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ListOperations as _, ObjectIO, ObjectOperations, ObjectToDelete};
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
@@ -104,7 +105,15 @@ fn encode_tier_delete_journal_entry(je: &Jentry) -> Result<Vec<u8>> {
pub async fn persist_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
where
S: ObjectIO,
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?;
save_config(api, &tier_delete_journal_object_name(je), data)
@@ -114,7 +123,14 @@ where
pub async fn remove_tier_delete_journal_entry<S>(api: Arc<S>, je: &Jentry) -> std::io::Result<()>
where
S: ObjectOperations,
S: ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
match delete_config(api, &tier_delete_journal_object_name(je)).await {
Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
@@ -19,13 +19,20 @@ use tokio_util::sync::CancellationToken;
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::Result;
use crate::object_api::ObjectInfo;
use crate::store::ECStore;
use crate::store_api::{ObjectInfo, ObjectInfoOrErr, WalkOptions};
use rustfs_storage_api::{BucketOperations, BucketOptions, ListOperations as _};
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{
BucketOperations, BucketOptions, ListOperations as _, ObjectInfoOrErr as StorageObjectInfoOrErr,
WalkOptions as StorageWalkOptions,
};
pub const DEFAULT_FREE_VERSION_RECOVERY_LIMIT: usize = 1_000;
const DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT: usize = 10_000;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, crate::error::Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FreeVersionRecoveryStats {
pub scanned: usize,
+56 -5
View File
@@ -17,11 +17,18 @@
use crate::bucket::metadata::BUCKET_METADATA_FILE;
use crate::bucket::replication::{decode_resync_file, encode_resync_file};
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::store_api::{ListOperations, ObjectIO, ObjectOperations, ObjectOptions, PutObjReader};
use crate::error::Error;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations};
use http::HeaderMap;
use rustfs_filemeta::FileInfo;
use rustfs_policy::auth::UserIdentity;
use rustfs_policy::policy::PolicyDoc;
use rustfs_storage_api::{BucketOperations, BucketOptions};
use rustfs_storage_api::{
BucketOperations, BucketOptions, DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectIO, ObjectInfoOrErr as StorageObjectInfoOrErr,
ObjectOperations, ObjectToDelete, WalkOptions as StorageWalkOptions,
};
use rustfs_utils::path::SLASH_SEPARATOR;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
@@ -40,6 +47,11 @@ const IAM_POLICY_DB_PREFIX: &str = "config/iam/policydb/";
const REPLICATION_META_DIR: &str = ".replication";
const RESYNC_META_FILE: &str = "resync.bin";
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
#[derive(Debug, Serialize, Deserialize)]
struct CompatIamFormat {
#[serde(default)]
@@ -179,7 +191,23 @@ fn normalize_bucket_meta_blob(path: &str, data: &[u8]) -> std::result::Result<Op
/// Skips buckets that already exist in RustFS (idempotent).
pub async fn try_migrate_bucket_metadata<S>(store: Arc<S>)
where
S: BucketOperations<Error = crate::error::Error> + ObjectIO + ObjectOperations,
S: BucketOperations<Error = crate::error::Error>
+ ObjectIO<
Error = crate::error::Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = crate::error::Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let buckets_list = match store
.list_bucket(&BucketOptions {
@@ -224,7 +252,7 @@ where
async fn migrate_one_if_missing<S>(store: Arc<S>, opts: &ObjectOptions, headers: &HeaderMap, path: &str, label: &str)
where
S: ObjectIO + ObjectOperations,
S: EcstoreObjectIO + EcstoreObjectOperations,
{
if store
.get_object_info(RUSTFS_META_BUCKET, path, &ObjectOptions::default())
@@ -278,7 +306,30 @@ where
/// If list_objects_v2 on the legacy bucket fails (e.g. format differs), migration is skipped.
pub async fn try_migrate_iam_config<S>(store: Arc<S>)
where
S: ListOperations + ObjectIO + ObjectOperations,
S: ListOperations<
Error = crate::error::Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = tokio_util::sync::CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + ObjectIO<
Error = crate::error::Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = crate::error::Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let opts = ObjectOptions {
max_parity: true,
@@ -14,7 +14,7 @@
use crate::bucket::metadata_sys::get_object_lock_config;
use crate::bucket::object_lock::objectlock;
use crate::store_api::ObjectInfo;
use crate::object_api::ObjectInfo;
use s3s::dto::{DefaultRetention, ObjectLockLegalHoldStatus, ObjectLockRetentionMode};
use std::sync::Arc;
use time::OffsetDateTime;
@@ -27,7 +27,8 @@ use crate::bucket::replication::replication_state::ReplicationStats;
use crate::config::com::{read_config, save_config};
use crate::disk::BUCKET_META_PREFIX;
use crate::error::Error as EcstoreError;
use crate::store_api::{ObjectIO, ObjectInfo, ObjectOptions};
use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::storage_api_contracts::EcstoreObjectIO;
use lazy_static::lazy_static;
use rustfs_filemeta::MrfOpKind;
use rustfs_filemeta::MrfReplicateEntry;
@@ -1252,7 +1253,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Returns `true` on success; on failure logs the error and returns `false`.
/// Callers must NOT clear their in-memory buffer on `false` so the next tick
/// can retry — otherwise a transient storage error permanently drops the batch.
async fn flush_mrf_to_disk<S: ObjectIO>(entries: &[MrfReplicateEntry], storage: &Arc<S>) -> bool {
async fn flush_mrf_to_disk<S: EcstoreObjectIO>(entries: &[MrfReplicateEntry], storage: &Arc<S>) -> bool {
match encode_mrf_file(entries) {
Ok(data) => {
if let Err(e) = save_config(storage.clone(), MRF_REPLICATION_FILE, data).await {
@@ -1281,7 +1282,7 @@ async fn flush_mrf_to_disk<S: ObjectIO>(entries: &[MrfReplicateEntry], storage:
}
/// Load bucket resync metadata from disk
async fn load_bucket_resync_metadata<S: ObjectIO>(
async fn load_bucket_resync_metadata<S: EcstoreObjectIO>(
bucket: &str,
obj_api: Arc<S>,
) -> Result<BucketReplicationResyncStatus, EcstoreError> {
@@ -31,9 +31,10 @@ use crate::error::{Error, Result, is_err_object_not_found, is_err_version_not_fo
use crate::event_notification::{EventArgs, send_event};
use crate::global::GLOBAL_LocalNodeName;
use crate::global::get_global_bucket_monitor;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::resolve_object_store_handle;
use crate::set_disk::get_lock_acquire_timeout;
use crate::store_api::{ListOperations, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, WalkOptions};
use crate::storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations};
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::operation::head_object::{HeadObjectError, HeadObjectOutput};
use aws_sdk_s3::primitives::ByteStream;
@@ -52,13 +53,17 @@ use http_body_util::StreamBody;
use regex::Regex;
use rmp_serde;
use rustfs_filemeta::{
MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo,
FileInfo, MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, ReplicateDecision, ReplicateObjectInfo,
ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType,
ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType,
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
};
use rustfs_s3_types::EventName;
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectToDelete};
use rustfs_storage_api::{
DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, NamespaceLocking as StorageNamespaceLocking, ObjectIO,
ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations, ObjectToDelete, WalkOptions as StorageWalkOptions,
};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, SUFFIX_OBJECTLOCK_LEGALHOLD_TIMESTAMP,
@@ -102,10 +107,67 @@ const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed";
const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed";
const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed";
/// Storage capabilities required by bucket replication workers.
pub trait ReplicationStorage: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking {}
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
impl<T> ReplicationStorage for T where T: ObjectIO + ObjectOperations + ListOperations + NamespaceLocking {}
/// Storage capabilities required by bucket replication workers.
pub trait ReplicationStorage:
ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
{
}
impl<T> ReplicationStorage for T where
T: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
{
}
pub(crate) const REPLICATION_DIR: &str = ".replication";
pub(crate) const RESYNC_FILE_NAME: &str = "resync.bin";
@@ -528,7 +590,18 @@ impl ReplicationResyncer {
}
}
pub async fn mark_status<S: ObjectIO>(&self, status: ResyncStatusType, opts: ResyncOpts, obj_layer: Arc<S>) -> Result<()> {
pub async fn mark_status<S>(&self, status: ResyncStatusType, opts: ResyncOpts, obj_layer: Arc<S>) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let bucket_status = {
let mut status_map = self.status_map.write().await;
let now = OffsetDateTime::now_utc();
@@ -641,7 +714,18 @@ impl ReplicationResyncer {
bucket_status.last_update = Some(now);
}
pub async fn persist_to_disk<S: ObjectIO>(&self, cancel_token: CancellationToken, api: Arc<S>) {
pub async fn persist_to_disk<S>(&self, cancel_token: CancellationToken, api: Arc<S>)
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let mut interval = tokio::time::interval(RESYNC_TIME_INTERVAL);
let mut last_update_times = HashMap::new();
@@ -694,7 +778,7 @@ impl ReplicationResyncer {
}
}
async fn resync_bucket_mark_status<S: ObjectIO>(&self, status: ResyncStatusType, opts: ResyncOpts, storage: Arc<S>) {
async fn resync_bucket_mark_status<S: EcstoreObjectIO>(&self, status: ResyncStatusType, opts: ResyncOpts, storage: Arc<S>) {
if let Err(err) = self.mark_status(status, opts.clone(), storage.clone()).await {
error!(
event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
@@ -1219,7 +1303,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
}
}
pub(crate) async fn save_resync_status<S: ObjectIO>(
pub(crate) async fn save_resync_status<S: EcstoreObjectIO>(
bucket: &str,
status: &BucketReplicationResyncStatus,
api: Arc<S>,
@@ -2181,7 +2265,7 @@ pub async fn replicate_delete<S: ReplicationStorage>(dobj: DeletedObjectReplicat
}
}
async fn source_delete_marker_missing<S: ObjectOperations>(
async fn source_delete_marker_missing<S: EcstoreObjectOperations>(
storage: &S,
bucket: &str,
object_name: &str,
@@ -2873,13 +2957,13 @@ pub async fn replicate_object<S: ReplicationStorage>(roi: ReplicateObjectInfo, s
}
trait ReplicateObjectInfoExt {
async fn replicate_object<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
async fn replicate_all<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
async fn replicate_object<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
async fn replicate_all<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo;
fn to_object_info(&self) -> ObjectInfo;
}
impl ReplicateObjectInfoExt for ReplicateObjectInfo {
async fn replicate_object<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
async fn replicate_object<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
let bucket = self.bucket.clone();
let object = self.name.clone();
@@ -3174,7 +3258,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
rinfo
}
async fn replicate_all<S: ObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
async fn replicate_all<S: EcstoreObjectIO>(&self, storage: Arc<S>, tgt_client: Arc<TargetClient>) -> ReplicatedTargetInfo {
let start_time = OffsetDateTime::now_utc();
let bucket = self.bucket.clone();
@@ -3896,7 +3980,7 @@ fn part_range_spec_from_actual_size(offset: i64, part_size: i64) -> std::io::Res
))
}
struct MultipartReplicationContext<'a, S: ObjectIO> {
struct MultipartReplicationContext<'a, S: EcstoreObjectIO> {
storage: Arc<S>,
cli: Arc<TargetClient>,
src_bucket: &'a str,
@@ -3908,7 +3992,7 @@ struct MultipartReplicationContext<'a, S: ObjectIO> {
put_opts: PutObjectOptions,
}
async fn replicate_object_with_multipart<S: ObjectIO>(ctx: MultipartReplicationContext<'_, S>) -> std::io::Result<()> {
async fn replicate_object_with_multipart<S: EcstoreObjectIO>(ctx: MultipartReplicationContext<'_, S>) -> std::io::Result<()> {
let MultipartReplicationContext {
storage,
cli,
+1 -1
View File
@@ -44,7 +44,7 @@ use crate::client::{
};
use crate::{
disk::DiskAPI,
store_api::{GetObjectReader, ObjectInfo},
object_api::{GetObjectReader, ObjectInfo},
};
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
+1 -1
View File
@@ -26,7 +26,7 @@ use std::collections::HashMap;
use crate::client::utils::base64_decode;
use crate::client::utils::base64_encode;
use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
use crate::{disk::DiskAPI, store_api::GetObjectReader};
use crate::{disk::DiskAPI, object_api::GetObjectReader};
use s3s::header::{
X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
};
@@ -25,7 +25,7 @@ use std::{collections::HashMap, io::Cursor, sync::Arc};
use tokio::io::BufReader;
use crate::error::ErrorResponse;
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use rustfs_filemeta::ObjectPartInfo;
use rustfs_rio::HashReader;
use rustfs_storage_api::HTTPRangeSpec;
@@ -24,8 +24,8 @@ use crate::bucket::lifecycle::lifecycle;
use crate::bucket::replication::{DeletedObjectReplicationInfo, check_replicate_delete, schedule_replication_delete};
use crate::bucket::versioning::VersioningApi;
use crate::bucket::versioning_sys::BucketVersioningSys;
use crate::object_api::ObjectOptions;
use crate::store::ECStore;
use crate::store_api::ObjectOptions;
use rustfs_filemeta::{REPLICATE_INCOMING_DELETE, ReplicationState, version_purge_statuses_map};
use rustfs_lock::MAX_DELETE_LIST;
use rustfs_storage_api::{ObjectOperations as _, ObjectToDelete};
+1 -1
View File
@@ -33,7 +33,7 @@ use crate::client::{
credentials::{CredContext, Credentials, SignatureType, Static},
signer_error,
};
use crate::{client::checksum::ChecksumMode, store_api::GetObjectReader};
use crate::{client::checksum::ChecksumMode, object_api::GetObjectReader};
use futures::{Future, StreamExt};
use http::{HeaderMap, HeaderName};
use http::{
+116 -19
View File
@@ -16,7 +16,8 @@ use crate::config::{audit, notify, oidc, set_global_storage_class, storageclass}
use crate::disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
use crate::error::{Error, Result};
use crate::global::is_first_cluster_node_local;
use crate::store_api::{ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, PutObjReader};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::storage_api_contracts::EcstoreObjectIO;
use http::HeaderMap;
use rustfs_config::audit::{
AUDIT_AMQP_KEYS, AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_KEYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_KEYS, AUDIT_MQTT_SUB_SYS,
@@ -32,7 +33,8 @@ use rustfs_config::notify::{
use rustfs_config::oidc::{IDENTITY_OPENID_KEYS, IDENTITY_OPENID_SUB_SYS, OIDC_REDIRECT_URI_DYNAMIC};
use rustfs_config::server_config::{Config, KVS};
use rustfs_config::{COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, EnableState, RUSTFS_REGION};
use rustfs_storage_api::StorageAdminApi;
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete, StorageAdminApi};
use rustfs_utils::path::SLASH_SEPARATOR;
use serde_json::{Map, Value};
use std::collections::{HashMap, HashSet};
@@ -182,12 +184,34 @@ fn audit_target_descriptors() -> [TargetConfigDescriptor; 9] {
}
#[instrument(skip(api))]
pub async fn read_config<S: ObjectIO>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
pub async fn read_config<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
Ok(data)
}
pub async fn read_config_no_lock<S: ObjectIO>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
pub async fn read_config_no_lock<S>(api: Arc<S>, file: &str) -> Result<Vec<u8>>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let (data, _obj) = read_config_with_metadata(
api,
file,
@@ -200,11 +224,18 @@ pub async fn read_config_no_lock<S: ObjectIO>(api: Arc<S>, file: &str) -> Result
Ok(data)
}
pub async fn read_config_with_metadata<S: ObjectIO>(
api: Arc<S>,
file: &str,
opts: &ObjectOptions,
) -> Result<(Vec<u8>, ObjectInfo)> {
pub async fn read_config_with_metadata<S>(api: Arc<S>, file: &str, opts: &ObjectOptions) -> Result<(Vec<u8>, ObjectInfo)>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let h = HeaderMap::new();
let mut rd = api
.get_object_reader(RUSTFS_META_BUCKET, file, None, h, opts)
@@ -228,7 +259,18 @@ pub async fn read_config_with_metadata<S: ObjectIO>(
}
#[instrument(skip(api, data))]
pub async fn save_config<S: ObjectIO>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
pub async fn save_config<S>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
save_config_with_opts(
api,
file,
@@ -242,7 +284,17 @@ pub async fn save_config<S: ObjectIO>(api: Arc<S>, file: &str, data: Vec<u8>) ->
}
#[instrument(skip(api))]
pub async fn delete_config<S: ObjectOperations>(api: Arc<S>, file: &str) -> Result<()> {
pub async fn delete_config<S>(api: Arc<S>, file: &str) -> Result<()>
where
S: ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
match api
.delete_object(
RUSTFS_META_BUCKET,
@@ -266,7 +318,18 @@ pub async fn delete_config<S: ObjectOperations>(api: Arc<S>, file: &str) -> Resu
}
}
pub async fn save_config_with_opts<S: ObjectIO>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
pub async fn save_config_with_opts<S>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let mut put_data = PutObjReader::from_vec(data);
if let Err(err) = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
error!("save_config_with_opts: err: {:?}, file: {}", err, file);
@@ -281,7 +344,7 @@ fn new_server_config() -> Config {
async fn new_and_save_server_config<S>(api: Arc<S>) -> Result<Config>
where
S: ObjectIO + StorageAdminApi,
S: EcstoreObjectIO + StorageAdminApi,
{
let mut cfg = new_server_config();
lookup_configs(&mut cfg, api.clone()).await;
@@ -985,7 +1048,22 @@ fn is_object_not_found(err: &Error) -> bool {
pub async fn try_migrate_server_config<S>(api: Arc<S>)
where
S: ObjectIO + ObjectOperations,
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let config_file = get_config_file();
match api
@@ -1069,7 +1147,7 @@ where
/// Handle the situation where the configuration file does not exist, create and save a new configuration
async fn handle_missing_config<S>(api: Arc<S>, context: &str) -> Result<Config>
where
S: ObjectIO + StorageAdminApi,
S: EcstoreObjectIO + StorageAdminApi,
{
warn!("Configuration not found ({}): Start initializing new configuration", context);
let cfg = if is_first_cluster_node_local().await {
@@ -1091,7 +1169,15 @@ fn handle_config_read_error(err: Error, file_path: &str) -> Result<Config> {
pub async fn read_config_without_migrate<S>(api: Arc<S>) -> Result<Config>
where
S: ObjectIO + StorageAdminApi,
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + StorageAdminApi,
{
let config_file = get_config_file();
@@ -1105,7 +1191,7 @@ where
async fn read_server_config<S>(api: Arc<S>, data: &[u8]) -> Result<Config>
where
S: ObjectIO + StorageAdminApi,
S: EcstoreObjectIO + StorageAdminApi,
{
// If the provided data is empty, try to read from the file again
if data.is_empty() {
@@ -1129,7 +1215,18 @@ where
Ok(cfg.merge())
}
pub async fn save_server_config<S: ObjectIO>(api: Arc<S>, cfg: &Config) -> Result<()> {
pub async fn save_server_config<S>(api: Arc<S>, cfg: &Config) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let config_file = get_config_file();
let existing = match read_config(api.clone(), &config_file).await {
Ok(v) => Some(v),
@@ -1216,8 +1313,8 @@ mod tests {
use crate::endpoints::SetupType;
use crate::error::{Error, Result};
use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::set_disk::SetDisks;
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use http::HeaderMap;
use rustfs_config::audit::{AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS};
use rustfs_config::notify::{
+1 -1
View File
@@ -13,8 +13,8 @@
// limitations under the License.
use crate::error::{Error, Result, is_err_data_movement_overwrite, is_err_object_not_found, is_err_version_not_found};
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::store::ECStore;
use crate::store_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use bytes::Bytes;
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
use rustfs_storage_api::{CompletePart, MultipartOperations as _, ObjectIO as _, ObjectOperations as _};
+1 -1
View File
@@ -715,7 +715,7 @@ pub fn cache_to_data_usage_info(cache: &DataUsageCache, path: &str, buckets: &[r
// Helper functions for DataUsageCache operations
pub async fn load_data_usage_cache(store: &crate::set_disk::SetDisks, name: &str) -> crate::error::Result<DataUsageCache> {
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::store_api::ObjectOptions;
use crate::object_api::ObjectOptions;
use http::HeaderMap;
use rand::RngExt;
use std::path::Path;
+1 -1
View File
@@ -17,8 +17,8 @@
use crate::bucket::metadata::BucketMetadata;
// use crate::event::name::EventName;
use crate::event::targetlist::TargetList;
use crate::object_api::ObjectInfo;
use crate::store::ECStore;
use crate::store_api::ObjectInfo;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::OnceLock;
+2 -1
View File
@@ -32,14 +32,15 @@ pub mod error;
pub mod global;
pub mod metrics_realtime;
pub mod notification_sys;
pub mod object_api;
pub mod pools;
pub mod rebalance;
pub mod rio;
pub mod rpc;
pub mod set_disk;
mod sets;
mod storage_api_contracts;
pub mod store;
pub mod store_api;
mod store_init;
pub mod store_list_objects;
pub mod store_utils;
@@ -21,14 +21,11 @@ use crate::store_utils::clean_metadata;
use crate::{bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent, bucket::lifecycle::lifecycle::TransitionOptions};
use bytes::Bytes;
use http::{HeaderMap, HeaderValue};
use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{
FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, ReplicateDecision, ReplicationState, ReplicationStatusType,
RestoreStatusOps as _, VersionPurgeStatusType, parse_restore_obj_status, replication_statuses_map,
version_purge_statuses_map,
};
use rustfs_lock::NamespaceLockWrapper;
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_rio::Checksum;
use rustfs_storage_api::{ExpirationOptions, HTTPRangeSpec, TransitionedObject};
use rustfs_utils::CompressionAlgorithm;
@@ -43,7 +40,6 @@ use std::sync::Arc;
use std::task::{Context, Poll};
use time::OffsetDateTime;
use tokio::io::{AsyncRead, AsyncReadExt, ReadBuf};
use tokio_util::sync::CancellationToken;
use tracing::warn;
use uuid::Uuid;
@@ -51,9 +47,7 @@ pub const ERASURE_ALGORITHM: &str = "rs-vandermonde";
pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
mod readers;
mod traits;
mod types;
pub use readers::*;
pub use traits::*;
pub use types::*;
@@ -4,14 +4,6 @@ use rustfs_storage_api::{
VersionMarker,
};
pub type ListObjectsInfo = rustfs_storage_api::ListObjectsInfo<ObjectInfo>;
pub type ListObjectsV2Info = rustfs_storage_api::ListObjectsV2Info<ObjectInfo>;
pub type ListObjectVersionsInfo = rustfs_storage_api::ListObjectVersionsInfo<ObjectInfo>;
pub type ObjectInfoOrErr = rustfs_storage_api::ObjectInfoOrErr<ObjectInfo, Error>;
pub type WalkOptions = rustfs_storage_api::WalkOptions<WalkFilter>;
pub type ObjectToDelete = rustfs_storage_api::ObjectToDelete;
pub type DeletedObject = rustfs_storage_api::DeletedObject;
#[derive(Debug, Default, Clone)]
pub struct ObjectOptions {
// Use the maximum parity (N/2), used when saving server configuration files
@@ -432,7 +424,7 @@ impl ObjectInfo {
restore_expires = restore_status.expiry();
}
// Convert parts from rustfs_filemeta::ObjectPartInfo to store_api::ObjectPartInfo
// Convert parts from rustfs_filemeta::ObjectPartInfo to object_api::ObjectPartInfo
let parts = fi
.parts
.iter()
@@ -715,9 +707,6 @@ fn versions_after_marker(file_infos: &rustfs_filemeta::FileInfoVersions, marker:
.map(|idx| &file_infos.versions[idx + 1..])
.unwrap_or(&file_infos.versions)
}
type WalkFilter = fn(&FileInfo) -> bool;
#[cfg(test)]
mod tests {
use super::*;
+2 -2
View File
@@ -36,9 +36,9 @@ use crate::error::{
is_err_operation_canceled, is_err_version_not_found,
};
use crate::notification_sys::get_global_notification_sys;
use crate::object_api::{GetObjectReader, ObjectOptions};
use crate::resolve_object_store_handle;
use crate::set_disk::SetDisks;
use crate::store_api::{GetObjectReader, ObjectOptions};
use crate::{global::GLOBAL_LifecycleSys, sets::Sets, store::ECStore};
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered};
@@ -1348,7 +1348,7 @@ pub(crate) async fn should_skip_lifecycle_for_data_movement(
};
let versioned = BucketVersioningSys::prefix_enabled(bucket, &version.name).await;
let object_info = crate::store_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
let object_info = crate::object_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
let event = eval_action_from_lifecycle(lifecycle_config, lock_retention, replication_config, &object_info).await;
match event.action {
+57 -12
View File
@@ -20,14 +20,17 @@ use crate::disk::error::DiskError;
use crate::error::{Error, Result};
use crate::error::{is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found, is_network_or_host_down};
use crate::global::get_global_endpoints;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use crate::pools::ListCallback;
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
use crate::storage_api_contracts::EcstoreObjectIO;
use crate::store::ECStore;
use crate::store_api::{GetObjectReader, NamespaceLocking, ObjectIO, ObjectInfo, ObjectOptions};
use http::HeaderMap;
use rand::RngExt as _;
use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _, StorageAdminApi};
use rustfs_storage_api::{
HTTPRangeSpec, NamespaceLocking as StorageNamespaceLocking, ObjectIO, ObjectOperations as _, StorageAdminApi,
};
use rustfs_utils::path::encode_dir_object;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
@@ -589,11 +592,33 @@ impl RebalanceMeta {
pub fn new() -> Self {
Self::default()
}
pub async fn load<S: ObjectIO>(&mut self, store: Arc<S>) -> Result<()> {
pub async fn load<S>(&mut self, store: Arc<S>) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
self.load_with_opts(store, ObjectOptions::default()).await
}
pub async fn load_with_opts<S: ObjectIO>(&mut self, store: Arc<S>, opts: ObjectOptions) -> Result<()> {
pub async fn load_with_opts<S>(&mut self, store: Arc<S>, opts: ObjectOptions) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let (data, _) = read_config_with_metadata(store, REBAL_META_NAME, &opts).await?;
if data.is_empty() {
debug!(
@@ -634,11 +659,33 @@ impl RebalanceMeta {
Ok(())
}
pub async fn save<S: ObjectIO>(&self, store: Arc<S>) -> Result<()> {
pub async fn save<S>(&self, store: Arc<S>) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
self.save_with_opts(store, ObjectOptions::default()).await
}
pub async fn save_with_opts<S: ObjectIO>(&self, store: Arc<S>, opts: ObjectOptions) -> Result<()> {
pub async fn save_with_opts<S>(&self, store: Arc<S>, opts: ObjectOptions) -> Result<()>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
if self.pool_stats.is_empty() {
debug!(
event = EVENT_REBALANCE_STATE,
@@ -667,12 +714,10 @@ impl RebalanceMeta {
}
impl ECStore {
async fn save_rebalance_meta_with_merge<S: ObjectIO + NamespaceLocking>(
&self,
pool: Arc<S>,
local_snapshot: &RebalanceMeta,
stage: &str,
) -> Result<()> {
async fn save_rebalance_meta_with_merge<S>(&self, pool: Arc<S>, local_snapshot: &RebalanceMeta, stage: &str) -> Result<()>
where
S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
{
let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
let _guard = ns_lock
.get_write_lock(get_lock_acquire_timeout())
+12 -6
View File
@@ -33,9 +33,7 @@ use crate::erasure_coding;
use crate::error::{Error, Result, is_err_version_not_found};
use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
use crate::global::{GLOBAL_LocalNodeName, GLOBAL_TierConfigMgr};
use crate::store_api::ListObjectVersionsInfo;
use crate::store_api::ObjectOptions;
use crate::store_api::{ObjectInfoOrErr, WalkOptions};
use crate::object_api::ObjectOptions;
use crate::{
bucket::lifecycle::bucket_lifecycle_ops::{
LifecycleOps, gen_transition_objname, get_transitioned_object_reader, put_restore_opts,
@@ -51,7 +49,7 @@ use crate::{
// event::name::EventName,
event_notification::{EventArgs, send_event},
global::{GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_deployment_id, is_dist_erasure},
store_api::{GetObjectReader, ListObjectsV2Info, MultipartOperations, ObjectIO, ObjectInfo, ObjectOperations, PutObjReader},
object_api::{GetObjectReader, ObjectInfo, PutObjReader},
store_init::load_format_erasure,
};
use bytes::Bytes;
@@ -82,11 +80,14 @@ use rustfs_object_capacity::capacity_scope::{
CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope,
};
use rustfs_s3_types::EventName;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
};
use rustfs_storage_api::{
HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
};
use rustfs_storage_api::{MultipartOperations as _, NamespaceLocking as _, ObjectIO as _, ObjectOperations as _};
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
@@ -131,6 +132,11 @@ use tracing::error;
use tracing::{Instrument, debug, info, warn};
use uuid::Uuid;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_SET_DISK: &str = "set_disk";
const EVENT_SET_DISK_MULTIPART: &str = "set_disk_multipart";
@@ -5014,7 +5020,7 @@ mod tests {
use crate::disk::health_state::RuntimeDriveHealthState;
use crate::endpoints::SetupType;
use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
use crate::store_api::ObjectInfo;
use crate::object_api::ObjectInfo;
use crate::store_init::save_format_file;
use crate::store_list_objects::ListPathOptions;
use rustfs_filemeta::ErasureInfo;
+10 -4
View File
@@ -15,7 +15,6 @@
use crate::disk::error_reduce::count_errs;
use crate::error::{Error, Result};
use crate::store_api::{ObjectInfoOrErr, WalkOptions};
use crate::{
disk::{
DiskAPI, DiskInfo, DiskOption, DiskStore,
@@ -26,8 +25,8 @@ use crate::{
endpoints::{Endpoints, PoolEndpoints},
error::StorageError,
global::{GLOBAL_LOCAL_DISK_SET_DRIVES, get_global_lock_clients, is_dist_erasure},
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
set_disk::SetDisks,
store_api::{GetObjectReader, ListObjectVersionsInfo, ListObjectsV2Info, ObjectInfo, ObjectOptions, PutObjReader},
store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file},
};
use futures::{
@@ -47,8 +46,10 @@ use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_storage_api::CompletePart;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DeletedObject, ListMultipartsInfo, ListPartsInfo,
MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListPartsInfo,
MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectToDelete, PartInfo,
WalkOptions as StorageWalkOptions,
};
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash};
@@ -61,6 +62,11 @@ use tracing::warn;
use tracing::{error, info};
use uuid::Uuid;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
#[derive(Debug, Clone)]
pub struct Sets {
pub id: Uuid,
+105
View File
@@ -0,0 +1,105 @@
use crate::error::Error;
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
use rustfs_filemeta::FileInfo;
use rustfs_storage_api::{
DeletedObject, HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectToDelete,
WalkOptions as StorageWalkOptions,
};
use std::fmt::Debug;
use tokio_util::sync::CancellationToken;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
pub(crate) trait EcstoreObjectIO:
rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
impl<T> EcstoreObjectIO for T where
T: rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
pub(crate) trait EcstoreObjectOperations:
rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
impl<T> EcstoreObjectOperations for T where
T: rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
pub(crate) trait EcstoreListOperations:
rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + Send
+ Sync
+ Debug
{
}
impl<T> EcstoreListOperations for T where
T: rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + Send
+ Sync
+ Debug
{
}
+10 -3
View File
@@ -53,16 +53,15 @@ use crate::notification_sys::get_global_notification_sys;
use crate::pools::PoolMeta;
use crate::rebalance::RebalanceMeta;
use crate::rpc::RemoteClient;
use crate::store_api::{ListObjectVersionsInfo, ObjectInfoOrErr, WalkOptions};
use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
use crate::tier::tier::TierConfigMgr;
use crate::{
bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk},
endpoints::EndpointServerPools,
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
rpc::S3PeerSys,
sets::Sets,
store_api::{GetObjectReader, ListObjectsV2Info, ObjectInfo, ObjectOptions, PutObjReader},
store_init,
};
use futures::future::join_all;
@@ -75,11 +74,14 @@ use rustfs_config::server_config::{Config, get_global_server_config, set_global_
use rustfs_filemeta::FileInfo;
use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::HTTPRangeSpec;
use rustfs_storage_api::{
BucketInfo, BucketOperations, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, ListMultipartsInfo,
ListPartsInfo, MakeBucketOptions, MultipartInfo, MultipartUploadResult, ObjectToDelete, PartInfo,
};
use rustfs_storage_api::{
HTTPRangeSpec, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
};
use rustfs_utils::path::{decode_dir_object, encode_dir_object, path_join_buf};
use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
use std::cmp::Ordering;
@@ -100,6 +102,11 @@ use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, instrument, warn};
use uuid::Uuid;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
/// Check if a directory contains any xl.meta files (indicating actual S3 objects)
/// This is used to determine if a bucket is empty for deletion purposes.
async fn has_xlmeta_files(path: &std::path::Path) -> bool {
-160
View File
@@ -1,160 +0,0 @@
use super::*;
use rustfs_storage_api::{
CompletePart, DeletedObject, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, ObjectToDelete,
PartInfo,
};
pub trait ObjectIO:
rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
impl<T> ObjectIO for T where
T: rustfs_storage_api::ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + Send
+ Sync
+ Debug
+ 'static
{
}
/// Object-level storage operations (beyond basic I/O in ObjectIO).
pub trait ObjectOperations:
rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
impl<T> ObjectOperations for T where
T: rustfs_storage_api::ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
> + Send
+ Sync
+ Debug
{
}
/// Listing and walking operations.
pub trait ListOperations:
rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + Send
+ Sync
+ Debug
{
}
impl<T> ListOperations for T where
T: rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
> + Send
+ Sync
+ Debug
{
}
/// Multipart upload operations.
pub trait MultipartOperations:
rustfs_storage_api::MultipartOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
PutObjectReader = PutObjReader,
CompletePart = CompletePart,
ListMultipartsInfo = ListMultipartsInfo,
MultipartUploadResult = MultipartUploadResult,
PartInfo = PartInfo,
MultipartInfo = MultipartInfo,
ListPartsInfo = ListPartsInfo,
> + Send
+ Sync
+ Debug
{
}
impl<T> MultipartOperations for T where
T: rustfs_storage_api::MultipartOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
PutObjectReader = PutObjReader,
CompletePart = CompletePart,
ListMultipartsInfo = ListMultipartsInfo,
MultipartUploadResult = MultipartUploadResult,
PartInfo = PartInfo,
MultipartInfo = MultipartInfo,
ListPartsInfo = ListPartsInfo,
> + Send
+ Sync
+ Debug
{
}
/// Healing and repair operations.
pub trait HealOperations:
rustfs_storage_api::HealOperations<Error = Error, HealResultItem = HealResultItem, HealOptions = HealOpts> + Send + Sync + Debug
{
}
impl<T> HealOperations for T where
T: rustfs_storage_api::HealOperations<Error = Error, HealResultItem = HealResultItem, HealOptions = HealOpts>
+ Send
+ Sync
+ Debug
{
}
/// Namespace lock operations needed by consumers that only coordinate object
/// mutations but do not require the full storage API surface.
pub trait NamespaceLocking:
rustfs_storage_api::NamespaceLocking<Error = Error, NamespaceLock = NamespaceLockWrapper> + Send + Sync + Debug + 'static
{
}
impl<T> NamespaceLocking for T where
T: rustfs_storage_api::NamespaceLocking<Error = Error, NamespaceLock = NamespaceLockWrapper> + Send + Sync + Debug + 'static
{
}
+13 -3
View File
@@ -21,17 +21,21 @@ use crate::disk::{DiskInfo, DiskStore};
use crate::error::{
Error, Result, StorageError, is_all_not_found, is_all_volume_not_found, is_err_bucket_not_found, to_object_err,
};
use crate::object_api::{ObjectInfo, ObjectOptions};
use crate::set_disk::SetDisks;
use crate::store_api::{ListObjectVersionsInfo, ListObjectsInfo, ObjectInfo, ObjectInfoOrErr, ObjectOptions, WalkOptions};
use crate::store::ECStore;
use crate::store_utils::is_reserved_or_invalid_bucket;
use crate::{store::ECStore, store_api::ListObjectsV2Info};
use futures::future::join_all;
use rand::seq::SliceRandom;
use rustfs_filemeta::{
MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
merge_file_meta_versions,
};
use rustfs_storage_api::{ObjectOperations as _, VersionMarker, WalkVersionsSortOrder};
use rustfs_storage_api::{
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo as StorageListObjectsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations as _,
VersionMarker, WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
};
use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
@@ -48,6 +52,12 @@ const MAX_OBJECT_LIST: i32 = 1000;
const METACACHE_SHARE_PREFIX: bool = false;
type ListObjectsInfo = StorageListObjectsInfo<ObjectInfo>;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
fn normalize_max_keys(max_keys: i32) -> i32 {
max_keys.min(MAX_OBJECT_LIST)
}
+70 -16
View File
@@ -49,11 +49,13 @@ use crate::{
config::com::{CONFIG_PREFIX, read_config},
disk::{MIGRATING_META_BUCKET, RUSTFS_META_BUCKET},
global::is_first_cluster_node_local,
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
storage_api_contracts::{EcstoreObjectIO, EcstoreObjectOperations},
store::ECStore,
store_api::{ObjectIO, ObjectOperations, ObjectOptions, PutObjReader},
};
use rustfs_filemeta::FileInfo;
use rustfs_rio::HashReader;
use rustfs_storage_api::{ObjectIO as _, ObjectOperations as _};
use rustfs_storage_api::{DeletedObject, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::S3ErrorCode;
@@ -1032,19 +1034,36 @@ impl TierConfigMgr {
self.save_tiering_config(api).await
}
pub async fn save_tiering_config<S: ObjectIO>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
pub async fn save_tiering_config<S>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
let data = encode_external_tiering_config_blob(self)?;
let config_file = tier_config_path(TIER_CONFIG_FILE);
self.save_config(api, &config_file, data).await
}
pub async fn save_config<S: ObjectIO>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
) -> std::result::Result<(), std::io::Error> {
pub async fn save_config<S>(&self, api: Arc<S>, file: &str, data: Bytes) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
self.save_config_with_opts(
api,
file,
@@ -1057,13 +1076,24 @@ impl TierConfigMgr {
.await
}
pub async fn save_config_with_opts<S: ObjectIO>(
pub async fn save_config_with_opts<S>(
&self,
api: Arc<S>,
file: &str,
data: Bytes,
opts: &ObjectOptions,
) -> std::result::Result<(), std::io::Error> {
) -> std::result::Result<(), std::io::Error>
where
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
debug!("save tier config:{}", file);
let mut put_data = PutObjReader::from_vec(data.to_vec());
let _ = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await?;
@@ -1099,7 +1129,10 @@ impl TierConfigMgr {
}
}
async fn new_and_save_tiering_config<S: ObjectIO>(api: Arc<S>) -> Result<TierConfigMgr> {
async fn new_and_save_tiering_config<S>(api: Arc<S>) -> Result<TierConfigMgr>
where
S: EcstoreObjectIO,
{
let mut cfg = TierConfigMgr {
driver_cache: HashMap::new(),
tiers: HashMap::new(),
@@ -1158,12 +1191,15 @@ async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMg
}
}
async fn read_tier_config_from_bucket<S: ObjectIO>(
async fn read_tier_config_from_bucket<S>(
api: Arc<S>,
bucket: &str,
path: &str,
opts: &ObjectOptions,
) -> io::Result<Option<Vec<u8>>> {
) -> io::Result<Option<Vec<u8>>>
where
S: EcstoreObjectIO,
{
let mut rd = match api.get_object_reader(bucket, path, None, HeaderMap::new(), opts).await {
Ok(v) => v,
Err(err) if is_err_config_not_found(&err) => return Ok(None),
@@ -1176,7 +1212,10 @@ async fn read_tier_config_from_bucket<S: ObjectIO>(
Ok(Some(data))
}
async fn write_tier_config_to_rustfs<S: ObjectIO>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()> {
async fn write_tier_config_to_rustfs<S>(api: Arc<S>, path: &str, data: Bytes) -> io::Result<()>
where
S: EcstoreObjectIO,
{
let mut put_data = PutObjReader::from_vec(data.to_vec());
api.put_object(
RUSTFS_META_BUCKET,
@@ -1194,7 +1233,22 @@ async fn write_tier_config_to_rustfs<S: ObjectIO>(api: Arc<S>, path: &str, data:
pub async fn try_migrate_tiering_config<S>(api: Arc<S>)
where
S: ObjectIO + ObjectOperations,
S: ObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
> + ObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
let target_path = tier_config_path(TIER_CONFIG_FILE);
if api
@@ -16,12 +16,25 @@ use rustfs_common::heal_channel::HealOpts;
use rustfs_ecstore::{
disk::DiskStore,
error::Error,
object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader},
store::ECStore,
store_api::{HealOperations, NamespaceLocking},
};
use rustfs_filemeta::FileInfo;
use rustfs_lock::NamespaceLockWrapper;
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_storage_api::{HealOperations as StorageHealOperations, NamespaceLocking as StorageNamespaceLocking, StorageAdminApi};
use rustfs_storage_api::{
CompletePart, DeletedObject, HTTPRangeSpec, HealOperations as StorageHealOperations, ListMultipartsInfo,
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListPartsInfo,
MultipartInfo, MultipartOperations as StorageMultipartOperations, MultipartUploadResult,
NamespaceLocking as StorageNamespaceLocking, ObjectIO as StorageObjectIO, ObjectInfoOrErr as StorageObjectInfoOrErr,
ObjectOperations as StorageObjectOperations, ObjectToDelete, PartInfo, StorageAdminApi, WalkOptions as StorageWalkOptions,
};
use tokio_util::sync::CancellationToken;
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
fn storage_admin_api_type_name<T>() -> &'static str
where
@@ -35,20 +48,6 @@ where
std::any::type_name::<T>()
}
fn namespace_locking_type_name<T>() -> &'static str
where
T: NamespaceLocking,
{
std::any::type_name::<T>()
}
fn heal_operations_type_name<T>() -> &'static str
where
T: HealOperations,
{
std::any::type_name::<T>()
}
fn storage_namespace_locking_type_name<T>() -> &'static str
where
T: StorageNamespaceLocking<Error = Error, NamespaceLock = NamespaceLockWrapper>,
@@ -56,6 +55,68 @@ where
std::any::type_name::<T>()
}
fn storage_object_io_type_name<T>() -> &'static str
where
T: StorageObjectIO<
Error = Error,
RangeSpec = HTTPRangeSpec,
HeaderMap = http::HeaderMap,
ObjectOptions = ObjectOptions,
ObjectInfo = ObjectInfo,
GetObjectReader = GetObjectReader,
PutObjectReader = PutObjReader,
>,
{
std::any::type_name::<T>()
}
fn storage_object_operations_type_name<T>() -> &'static str
where
T: StorageObjectOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
FileInfo = FileInfo,
ObjectToDelete = ObjectToDelete,
DeletedObject = DeletedObject,
>,
{
std::any::type_name::<T>()
}
fn storage_list_operations_type_name<T>() -> &'static str
where
T: rustfs_storage_api::ListOperations<
Error = Error,
ListObjectsV2Info = ListObjectsV2Info,
ListObjectVersionsInfo = ListObjectVersionsInfo,
ObjectInfoOrErr = ObjectInfoOrErr,
WalkOptions = WalkOptions,
WalkCancellation = CancellationToken,
WalkResultSender = tokio::sync::mpsc::Sender<ObjectInfoOrErr>,
>,
{
std::any::type_name::<T>()
}
fn storage_multipart_operations_type_name<T>() -> &'static str
where
T: StorageMultipartOperations<
Error = Error,
ObjectInfo = ObjectInfo,
ObjectOptions = ObjectOptions,
PutObjectReader = PutObjReader,
CompletePart = CompletePart,
ListMultipartsInfo = ListMultipartsInfo,
MultipartUploadResult = MultipartUploadResult,
PartInfo = PartInfo,
MultipartInfo = MultipartInfo,
ListPartsInfo = ListPartsInfo,
>,
{
std::any::type_name::<T>()
}
fn storage_heal_operations_type_name<T>() -> &'static str
where
T: StorageHealOperations<Error = Error, HealResultItem = HealResultItem, HealOptions = HealOpts>,
@@ -68,21 +129,31 @@ fn ecstore_implements_storage_admin_api_contract() {
assert!(storage_admin_api_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_namespace_locking_contract() {
assert!(namespace_locking_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_heal_operations_contract() {
assert!(heal_operations_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_storage_namespace_locking_contract() {
assert!(storage_namespace_locking_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_storage_object_io_contract() {
assert!(storage_object_io_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_storage_object_operations_contract() {
assert!(storage_object_operations_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_storage_list_operations_contract() {
assert!(storage_list_operations_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_storage_multipart_operations_contract() {
assert!(storage_multipart_operations_type_name::<ECStore>().ends_with("::ECStore"));
}
#[test]
fn ecstore_implements_storage_heal_operations_contract() {
assert!(storage_heal_operations_type_name::<ECStore>().ends_with("::ECStore"));
@@ -8,7 +8,7 @@ use rustfs_ecstore::bitrot::create_bitrot_reader;
use rustfs_ecstore::disk::endpoint::Endpoint;
use rustfs_ecstore::disk::{DiskAPI as _, DiskOption, new_disk};
use rustfs_ecstore::erasure_coding::Erasure;
use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use rustfs_ecstore::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
use serde::Deserialize;
use sha2::{Digest, Sha256};
+3 -3
View File
@@ -29,6 +29,6 @@ pub(crate) use rustfs_ecstore::global::GLOBAL_LOCAL_DISK_MAP;
#[cfg(test)]
pub(crate) use rustfs_ecstore::disk::{DiskOption, new_disk};
pub type HealObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub type HealObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub type HealPutObjReader = rustfs_ecstore::store_api::PutObjReader;
pub type HealObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
pub type HealObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
pub type HealPutObjReader = rustfs_ecstore::object_api::PutObjReader;
+2 -2
View File
@@ -17,8 +17,8 @@ use std::sync::Arc;
pub(crate) const IAM_CONFIG_ROOT_PREFIX: &str = rustfs_ecstore::config::RUSTFS_CONFIG_PREFIX;
pub(crate) type IamEcstoreError = rustfs_ecstore::error::Error;
pub(crate) type IamConfigObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub(crate) type IamConfigObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub(crate) type IamConfigObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
pub(crate) type IamConfigObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
pub(crate) type IamStorageError = rustfs_ecstore::error::StorageError;
pub(crate) type IamStorageResult<T> = rustfs_ecstore::error::Result<T>;
pub(crate) type IamStore = rustfs_ecstore::store::ECStore;
+2 -2
View File
@@ -12,5 +12,5 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(super) type IamObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub(super) type IamObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub(super) type IamObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
pub(super) type IamObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
+1 -1
View File
@@ -18,7 +18,7 @@ use rustfs_ecstore::{config, global};
use crate::event::NotifyObjectInfo;
type EcstoreObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
type EcstoreObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
#[derive(Debug)]
pub(crate) enum NotifyConfigStoreError {
+4 -4
View File
@@ -18,10 +18,10 @@ pub(super) type SwiftBucketMetadata = rustfs_ecstore::bucket::metadata::BucketMe
pub(super) type SwiftStorageResult<T> = rustfs_ecstore::error::Result<T>;
pub(super) type SwiftStore = rustfs_ecstore::store::ECStore;
pub type SwiftGetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
pub type SwiftObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub type SwiftObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub type SwiftPutObjReader = rustfs_ecstore::store_api::PutObjReader;
pub type SwiftGetObjectReader = rustfs_ecstore::object_api::GetObjectReader;
pub type SwiftObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
pub type SwiftObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
pub type SwiftPutObjReader = rustfs_ecstore::object_api::PutObjReader;
pub fn resolve_swift_object_store_handle() -> Option<Arc<SwiftStore>> {
rustfs_ecstore::resolve_object_store_handle()
+3 -3
View File
@@ -12,9 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) type SelectGetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
pub(crate) type SelectObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub(crate) type SelectObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub(crate) type SelectGetObjectReader = rustfs_ecstore::object_api::GetObjectReader;
pub(crate) type SelectObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
pub(crate) type SelectObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
pub(crate) type SelectStorageError = rustfs_ecstore::error::StorageError;
pub(crate) type SelectStore = rustfs_ecstore::store::ECStore;
+4 -4
View File
@@ -58,11 +58,11 @@ pub(crate) use rustfs_ecstore::config::init as init_ecstore_config_for_scanner_t
#[cfg(test)]
pub(crate) use rustfs_ecstore::disk::{DiskOption, endpoint::Endpoint, new_disk};
pub type ScannerGetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
pub type ScannerObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub type ScannerObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub type ScannerGetObjectReader = rustfs_ecstore::object_api::GetObjectReader;
pub type ScannerObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
pub type ScannerObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
pub type ScannerObjectToDelete = ObjectToDelete;
pub type ScannerPutObjReader = rustfs_ecstore::store_api::PutObjReader;
pub type ScannerPutObjReader = rustfs_ecstore::object_api::PutObjReader;
pub(crate) fn resolve_scanner_object_store_handle() -> Option<Arc<ECStore>> {
rustfs_ecstore::resolve_object_store_handle()
+115 -11
View File
@@ -5,19 +5,17 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-foreground-scanner-admission-snapshots`
- Baseline: `origin/main` after `rustfs/rustfs#3606`
(`7cb7aefc3bfd33cfaa46c718db15f00b0471fe88`).
- Branch: `overtrue/arch-ecstore-object-api-alias-prune`
- Baseline: stacked after `rustfs/rustfs#3614`
(`3740e65246aed8c8ee9685ed0234997c7d54e1b5`).
- PR type for this branch: `consumer-migration`
- Runtime behavior changes: none.
- Rust code changes: extend the RustFS workload admission registry from
repair/replication snapshots to foreground-read, scanner, and metadata owner
snapshots.
- CI/script changes: extend migration guard coverage for RustFS workload
admission provider implementations.
- Docs changes: add RustFS runtime owner provider notes to
[`workload-admission-contracts.md`](workload-admission-contracts.md) and
record the API-059/R-019 provider slice.
- Rust code changes: remove unused public `rustfs-storage-api` passthrough
aliases from ECStore `object_api` and bind the ECStore contract test directly
to the storage-api generic list/delete/walk contracts.
- CI/script changes: make the migration guard reject restoring ECStore
`object_api` storage-api passthrough aliases.
- Docs changes: record the API-066 ECStore object API alias prune slice.
## Phase 0 Tasks
@@ -214,6 +212,112 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
check, migration and layer guards, formatting, diff hygiene, risk scan, and
three-expert review.
- [x] `API-060` Remove heal and namespace-lock operation compatibility facades.
- Completed slice: remove the old ECStore `store_api::HealOperations` and
`store_api::NamespaceLocking` compatibility subtraits after ECStore storage
types already implemented the shared `rustfs_storage_api` contracts
directly.
- Acceptance: internal ECStore bounds and compile-time coverage use the
shared storage-api heal and namespace-lock contracts directly, while the
remaining object/list/multipart compatibility bindings stay unchanged for
their active internal consumers.
- Must preserve: heal operation behavior, namespace-lock acquisition,
replication resync locking, rebalance metadata locking, object I/O,
multipart, list, and storage hot paths.
- Verification: focused ECStore contract tests, focused ECStore library
check, migration and layer guards, formatting, diff hygiene, risk scan, and
three-expert review.
- [x] `API-061` Remove public ECStore object operation compatibility facades.
- Completed slice: remove the old public ECStore `store_api` object, list,
and multipart operation compatibility subtraits, and keep internal generic
bounds on crate-private storage-api contract constraints instead of public
downstream compatibility traits.
- Acceptance: `store_api` no longer exports public operation compatibility
traits, ECStore direct storage-api compile-time coverage includes object,
object-operation, list, multipart, namespace-lock, heal, and admin
contracts, and remaining public `store_api` exports are DTO/reader
compatibility paths only.
- Must preserve: object I/O, list/walk behavior, multipart behavior, config
persistence, tier config migration, rebalance metadata locking, lifecycle
journal handling, replication MRF/resync persistence, and downstream DTO
import compatibility.
- Verification: focused ECStore contract tests, focused ECStore library
check, migration and layer guards, formatting, diff hygiene, risk scan, and
three-expert review.
- [x] `API-062` Establish explicit ECStore object API boundary.
- Completed slice: add `rustfs_ecstore::object_api` as the explicit public
path for ECStore-owned object DTO and reader contracts, then migrate
RustFS, scanner, heal, IAM, Swift, S3 Select, notify, and ECStore
integration-test compatibility aliases away from the legacy public
`store_api` path.
- Acceptance: external compatibility boundary modules no longer reference
`rustfs_ecstore::store_api` for ECStore-owned object DTO and reader
aliases, while `store_api` remains available only as the old internal
implementation module pending final compatibility removal.
- Must preserve: object metadata shape, option defaults, reader/writer
behavior, Swift/scanner/heal/IAM/S3 Select/notify boundary semantics, and
all storage hot paths.
- Verification: focused ECStore/RustFS/downstream compile checks, migration
guard, formatting, diff hygiene, risk scan, and three-expert review.
- [x] `API-063` Make legacy ECStore store API module private.
- Completed slice: remove `rustfs_ecstore::store_api` from the public crate
module surface after external compatibility boundaries moved to
`rustfs_ecstore::object_api`.
- Acceptance: ECStore object DTO and reader compatibility remains available
through `object_api`, integration contract tests consume the new public
path, and migration rules reject restoring `pub mod store_api`.
- Must preserve: internal ECStore object DTO definitions, reader/writer
behavior, storage-api trait bindings, and downstream object/list/multipart
compile-time contracts.
- Verification: focused ECStore contract tests, migration guard, formatting,
diff hygiene, risk scan, and three-expert review.
- [x] `API-064` Retire the ECStore store API module name.
- Completed slice: move ECStore object DTO, reader, and option definitions
from the private `store_api` module into the public `object_api` module,
then migrate ECStore internal imports to `crate::object_api`.
- Acceptance: no ECStore `store_api` module file or directory remains, public
consumers keep using `rustfs_ecstore::object_api`, and migration rules
reject restoring the retired module path.
- Must preserve: object metadata shape, reader/writer behavior, storage-api
contract bindings, object/list/multipart behavior, and downstream public
object API compatibility.
- Verification: focused ECStore compile checks, migration guard, formatting,
diff hygiene, risk scan, and three-expert review.
- [x] `API-065` Use storage-api list contracts inside ECStore.
- Completed slice: migrate ECStore internal list response, walk options, and
walk result bindings to local aliases over the generic `rustfs-storage-api`
contracts, including replication worker trait bounds, while retaining
public `rustfs_ecstore::object_api` aliases for downstream compatibility.
- Acceptance: ECStore implementation modules no longer import list/walk
compatibility aliases from `crate::object_api`, and migration rules reject
reintroducing those internal imports.
- Must preserve: list response shape, walk result item shape, object metadata
shape, storage-api trait bindings, and downstream public object API
compatibility.
- Verification: focused ECStore compile checks, migration guard, formatting,
diff hygiene, risk scan, and three-expert review.
- [x] `API-066` Prune ECStore object API storage aliases.
- Completed slice: remove unused public storage-api passthrough aliases from
ECStore `object_api` for list responses, walk options, walk result items,
and delete-object DTOs, then bind the ECStore contract test directly to the
generic `rustfs-storage-api` contracts.
- Acceptance: ECStore `object_api` no longer exposes storage-api passthrough
aliases, the storage contract test still proves ECStore implements the
storage-api traits with the same associated concrete types, and migration
rules reject restoring the object_api passthrough aliases.
- Must preserve: ECStore-owned object metadata, object options, reader/writer
types, storage-api trait associated type bindings, list/delete/walk response
shapes, and runtime behavior.
- Verification: focused ECStore compile checks, storage contract test,
downstream compile checks, migration and layer guards, formatting, diff
hygiene, risk scan, full pre-commit, and three-expert review.
- [x] `TEST-PRTYPE-001` Check PR type enum consistency.
- Acceptance: `./scripts/check_architecture_migration_rules.sh` parses the
allowed PR types from [`crate-boundaries.md`](crate-boundaries.md) and fails
+4 -4
View File
@@ -39,7 +39,7 @@ pub(crate) use rustfs_ecstore::rpc::{
pub(crate) use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk_path, find_local_disk_by_ref};
pub(crate) type GetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
pub(crate) type ObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub(crate) type ObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub(crate) type PutObjReader = rustfs_ecstore::store_api::PutObjReader;
pub(crate) type GetObjectReader = rustfs_ecstore::object_api::GetObjectReader;
pub(crate) type ObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
pub(crate) type ObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
pub(crate) type PutObjReader = rustfs_ecstore::object_api::PutObjReader;
+89 -45
View File
@@ -57,6 +57,8 @@ STORE_API_OBJECT_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_object_helper_reexp
STORE_API_RANGE_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_range_helper_reexports.txt"
STORE_API_LIST_HELPER_REEXPORTS_FILE="${TMP_DIR}/store_api_list_helper_reexports.txt"
STORE_API_LIST_RESPONSE_REEXPORTS_FILE="${TMP_DIR}/store_api_list_response_reexports.txt"
ECSTORE_OBJECT_API_LIST_ALIAS_INTERNAL_HITS_FILE="${TMP_DIR}/ecstore_object_api_list_alias_internal_hits.txt"
ECSTORE_OBJECT_API_STORAGE_ALIAS_HITS_FILE="${TMP_DIR}/ecstore_object_api_storage_alias_hits.txt"
STORE_API_DELETE_DTO_REEXPORTS_FILE="${TMP_DIR}/store_api_delete_dto_reexports.txt"
STORE_API_DELETE_DTO_INTERNAL_HITS_FILE="${TMP_DIR}/store_api_delete_dto_internal_hits.txt"
STORE_API_LIFECYCLE_HELPER_DEFINITION_HITS_FILE="${TMP_DIR}/store_api_lifecycle_helper_definition_hits.txt"
@@ -73,6 +75,8 @@ PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE="${TMP_DIR}/production_unused_compat_al
BROAD_STORE_API_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/broad_store_api_compat_reexport_hits.txt"
NESTED_STORE_API_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/nested_store_api_compat_module_hits.txt"
UNAPPROVED_STORE_API_COMPAT_ALIAS_HITS_FILE="${TMP_DIR}/unapproved_store_api_compat_alias_hits.txt"
PUBLIC_STORE_API_MODULE_HITS_FILE="${TMP_DIR}/public_store_api_module_hits.txt"
STORE_API_MODULE_PATH_HITS_FILE="${TMP_DIR}/store_api_module_path_hits.txt"
ECSTORE_COMPAT_PASSTHROUGH_EXPECTED_FILE="${TMP_DIR}/ecstore_compat_passthrough_expected.txt"
ECSTORE_COMPAT_PASSTHROUGH_ACTUAL_FILE="${TMP_DIR}/ecstore_compat_passthrough_actual.txt"
ECSTORE_COMPAT_PASSTHROUGH_DIFF_FILE="${TMP_DIR}/ecstore_compat_passthrough_diff.txt"
@@ -357,7 +361,7 @@ fi
(
cd "$ROOT_DIR"
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api::\{[^}]*\b(?:BucketInfo|BucketOptions|DeleteBucketOptions|MakeBucketOptions|SRBucketDeleteOp)\b' \
crates/ecstore/src/store_api.rs || true
crates/ecstore/src/store_api.rs 2>/dev/null || true
) >"$STORE_API_BUCKET_DTO_REEXPORTS_FILE"
if [[ -s "$STORE_API_BUCKET_DTO_REEXPORTS_FILE" ]]; then
@@ -366,8 +370,17 @@ fi
(
cd "$ROOT_DIR"
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api::\{[^}]*\bBucketOperations\b|pub trait BucketOperations\b' \
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/traits.rs || true
store_api_bucket_operation_scan_targets=()
if [[ -f crates/ecstore/src/store_api.rs ]]; then
store_api_bucket_operation_scan_targets+=(crates/ecstore/src/store_api.rs)
fi
if [[ -f crates/ecstore/src/store_api/traits.rs ]]; then
store_api_bucket_operation_scan_targets+=(crates/ecstore/src/store_api/traits.rs)
fi
if (( ${#store_api_bucket_operation_scan_targets[@]} > 0 )); then
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api::\{[^}]*\bBucketOperations\b|pub trait BucketOperations\b' \
"${store_api_bucket_operation_scan_targets[@]}" || true
fi
) >"$STORE_API_BUCKET_OPERATION_HITS_FILE"
if [[ -s "$STORE_API_BUCKET_OPERATION_HITS_FILE" ]]; then
@@ -377,7 +390,7 @@ fi
(
cd "$ROOT_DIR"
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api::\{[^}]*\b(?:CompletePart|ListMultipartsInfo|ListPartsInfo|MultipartInfo|MultipartUploadResult|PartInfo)\b|pub struct (?:CompletePart|ListMultipartsInfo|ListPartsInfo|MultipartInfo|MultipartUploadResult|PartInfo)\b' \
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs 2>/dev/null || true
) >"$STORE_API_MULTIPART_DTO_REEXPORTS_FILE"
if [[ -s "$STORE_API_MULTIPART_DTO_REEXPORTS_FILE" ]]; then
@@ -387,7 +400,7 @@ fi
(
cd "$ROOT_DIR"
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:HTTPPreconditions|ObjectLockRetentionOptions|ObjectPreconditionError|ObjectPreconditionPart|ObjectPreconditionState)\b|::(?:HTTPPreconditions|ObjectLockRetentionOptions|ObjectPreconditionError|ObjectPreconditionPart|ObjectPreconditionState)\b)|pub (?:enum ObjectPreconditionError|struct (?:HTTPPreconditions|ObjectLockRetentionOptions|ObjectPreconditionPart|ObjectPreconditionState))\b' \
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs 2>/dev/null || true
) >"$STORE_API_OBJECT_HELPER_REEXPORTS_FILE"
if [[ -s "$STORE_API_OBJECT_HELPER_REEXPORTS_FILE" ]]; then
@@ -397,7 +410,7 @@ fi
(
cd "$ROOT_DIR"
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:HTTPRangeError|HTTPRangeSpec)\b|::(?:HTTPRangeError|HTTPRangeSpec)\b)|pub (?:enum HTTPRangeError|struct HTTPRangeSpec)\b' \
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs crates/ecstore/src/store_api/readers.rs || true
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs crates/ecstore/src/store_api/readers.rs 2>/dev/null || true
) >"$STORE_API_RANGE_HELPER_REEXPORTS_FILE"
if [[ -s "$STORE_API_RANGE_HELPER_REEXPORTS_FILE" ]]; then
@@ -407,7 +420,7 @@ fi
(
cd "$ROOT_DIR"
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:VersionMarker|WalkOptions|WalkVersionsSortOrder)\b|::(?:VersionMarker|WalkOptions|WalkVersionsSortOrder)\b)|pub (?:enum (?:VersionMarker|WalkVersionsSortOrder)|struct WalkOptions)\b' \
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs 2>/dev/null || true
) >"$STORE_API_LIST_HELPER_REEXPORTS_FILE"
if [[ -s "$STORE_API_LIST_HELPER_REEXPORTS_FILE" ]]; then
@@ -417,7 +430,7 @@ fi
(
cd "$ROOT_DIR"
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ListOperations|ObjectInfoOrErr)\b|::(?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ListOperations|ObjectInfoOrErr)\b)|pub struct (?:ListObjectVersionsInfo|ListObjectsInfo|ListObjectsV2Info|ObjectInfoOrErr)\b' \
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs 2>/dev/null || true
) >"$STORE_API_LIST_RESPONSE_REEXPORTS_FILE"
if [[ -s "$STORE_API_LIST_RESPONSE_REEXPORTS_FILE" ]]; then
@@ -427,7 +440,7 @@ fi
(
cd "$ROOT_DIR"
rg -n --no-heading 'pub(?:\(crate\))? use rustfs_storage_api(?:::\{[^}]*\b(?:DeletedObject|ObjectToDelete)\b|::(?:DeletedObject|ObjectToDelete)\b)|pub struct (?:DeletedObject|ObjectToDelete)\b' \
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs || true
crates/ecstore/src/store_api.rs crates/ecstore/src/store_api/types.rs 2>/dev/null || true
) >"$STORE_API_DELETE_DTO_REEXPORTS_FILE"
if [[ -s "$STORE_API_DELETE_DTO_REEXPORTS_FILE" ]]; then
@@ -484,6 +497,34 @@ if [[ -s "$STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE" ]]; then
report_failure "external list response consumers must use rustfs-storage-api contracts: $(paste -sd '; ' "$STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
find crates/ecstore/src -type f -name '*.rs' -print0 |
xargs -0 perl -0ne '
while (/crate::object_api(?:::\{[^}]*\b(?:ListObjectsInfo|ListObjectsV2Info|ListObjectVersionsInfo|ObjectInfoOrErr|WalkOptions)\b|::(?:ListObjectsInfo|ListObjectsV2Info|ListObjectVersionsInfo|ObjectInfoOrErr|WalkOptions)\b)/sg) {
my $prefix = substr($_, 0, $-[0]);
my $line = ($prefix =~ tr/\n//) + 1;
my $match = $&;
$match =~ s/\s+/ /g;
print "$ARGV:$line:$match\n";
}
' || true
) >"$ECSTORE_OBJECT_API_LIST_ALIAS_INTERNAL_HITS_FILE"
if [[ -s "$ECSTORE_OBJECT_API_LIST_ALIAS_INTERNAL_HITS_FILE" ]]; then
report_failure "ECStore internal list/walk consumers must bind rustfs-storage-api generic contracts directly: $(paste -sd '; ' "$ECSTORE_OBJECT_API_LIST_ALIAS_INTERNAL_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --no-heading 'pub type (?:ListObjectsInfo|ListObjectsV2Info|ListObjectVersionsInfo|ObjectInfoOrErr|WalkOptions|ObjectToDelete|DeletedObject)\b' \
crates/ecstore/src/object_api/types.rs || true
) >"$ECSTORE_OBJECT_API_STORAGE_ALIAS_HITS_FILE"
if [[ -s "$ECSTORE_OBJECT_API_STORAGE_ALIAS_HITS_FILE" ]]; then
report_failure "ECStore object_api must not re-export storage-api passthrough aliases: $(paste -sd '; ' "$ECSTORE_OBJECT_API_STORAGE_ALIAS_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --no-heading 'rustfs_ecstore::store_api(?:::\{[^}]*\b(?:ObjectIO|ObjectOperations|ListOperations|MultipartOperations|HealOperations|NamespaceLocking)\b|::(?:ObjectIO|ObjectOperations|ListOperations|MultipartOperations|HealOperations|NamespaceLocking)\b)' \
@@ -613,14 +654,27 @@ fi
--glob '*storage_compat.rs' \
--glob '!crates/ecstore/**' \
--glob '!crates/e2e_test/**' || true
) | awk '
$0 !~ /^[^:]+:[0-9]+:[[:space:]]*(pub(\([^)]*\))?[[:space:]]+)?type[[:space:]]+[A-Za-z0-9_]+[[:space:]]*=[[:space:]]*rustfs_ecstore::store_api::(GetObjectReader|ObjectInfo|ObjectOptions|PutObjReader);$/ {
print
}
' >"$UNAPPROVED_STORE_API_COMPAT_ALIAS_HITS_FILE"
) >"$UNAPPROVED_STORE_API_COMPAT_ALIAS_HITS_FILE"
if [[ -s "$UNAPPROVED_STORE_API_COMPAT_ALIAS_HITS_FILE" ]]; then
report_failure "storage compatibility boundaries may only keep explicit ECStore-owned store_api type aliases: $(paste -sd '; ' "$UNAPPROVED_STORE_API_COMPAT_ALIAS_HITS_FILE")"
report_failure "storage compatibility boundaries must use rustfs_ecstore::object_api for ECStore-owned object DTO and reader aliases: $(paste -sd '; ' "$UNAPPROVED_STORE_API_COMPAT_ALIAS_HITS_FILE")"
fi
if rg -n --no-heading '^\s*pub\s+mod\s+store_api\s*;' "$ROOT_DIR/crates/ecstore/src/lib.rs" >"$PUBLIC_STORE_API_MODULE_HITS_FILE"; then
report_failure "ECStore store_api must remain private; expose ECStore-owned object DTO and reader aliases through rustfs_ecstore::object_api"
fi
(
cd "$ROOT_DIR"
{
[[ -e crates/ecstore/src/store_api.rs ]] && printf '%s\n' 'crates/ecstore/src/store_api.rs'
[[ -d crates/ecstore/src/store_api ]] && printf '%s\n' 'crates/ecstore/src/store_api'
true
}
) >"$STORE_API_MODULE_PATH_HITS_FILE"
if [[ -s "$STORE_API_MODULE_PATH_HITS_FILE" ]]; then
report_failure "legacy ECStore store_api module files must not be restored: $(paste -sd '; ' "$STORE_API_MODULE_PATH_HITS_FILE")"
fi
cat >"$ECSTORE_COMPAT_PASSTHROUGH_EXPECTED_FILE" <<'EOF'
@@ -652,50 +706,40 @@ fi
(
cd "$ROOT_DIR"
rg -n --no-heading 'async fn (get_object_reader|put_object|get_object_info|verify_object_integrity|copy_object|delete_object_version|delete_object|delete_objects|put_object_metadata|get_object_tags|put_object_tags|delete_object_tags|add_partial|transition_object|restore_transitioned_object|list_multipart_uploads|new_multipart_upload|copy_object_part|put_object_part|get_multipart_info|list_object_parts|abort_multipart_upload|complete_multipart_upload|heal_format|heal_bucket|heal_object|get_pool_and_set|check_abandoned_parts|new_ns_lock)\b' \
crates/ecstore/src/store_api/traits.rs || true
if [[ -f crates/ecstore/src/store_api/traits.rs ]]; then
rg -n --no-heading 'async fn (get_object_reader|put_object|get_object_info|verify_object_integrity|copy_object|delete_object_version|delete_object|delete_objects|put_object_metadata|get_object_tags|put_object_tags|delete_object_tags|add_partial|transition_object|restore_transitioned_object|list_multipart_uploads|new_multipart_upload|copy_object_part|put_object_part|get_multipart_info|list_object_parts|abort_multipart_upload|complete_multipart_upload|heal_format|heal_bucket|heal_object|get_pool_and_set|check_abandoned_parts|new_ns_lock)\b' \
crates/ecstore/src/store_api/traits.rs || true
fi
) >"$STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE"
if [[ -s "$STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE" ]]; then
report_failure "old ecstore operation method signatures reintroduced: $(paste -sd '; ' "$STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE")"
fi
require_source_contains \
"crates/ecstore/src/store_api/traits.rs" \
"rustfs_storage_api::ObjectIO<" \
"ECStore ObjectIO compatibility binding"
require_source_contains \
"crates/ecstore/src/store_api/traits.rs" \
"rustfs_storage_api::ObjectOperations<" \
"ECStore ObjectOperations compatibility binding"
require_source_contains \
"crates/ecstore/src/store_api/traits.rs" \
"rustfs_storage_api::MultipartOperations<" \
"ECStore MultipartOperations compatibility binding"
require_source_contains \
"crates/ecstore/src/store_api/traits.rs" \
"rustfs_storage_api::HealOperations<" \
"ECStore HealOperations compatibility binding"
require_source_contains \
"crates/ecstore/src/store_api/traits.rs" \
"rustfs_storage_api::NamespaceLocking<" \
"ECStore NamespaceLocking compatibility binding"
require_source_contains \
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
"fn ecstore_implements_storage_admin_api_contract()" \
"ECStore StorageAdminApi compile-time coverage test"
require_source_contains \
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
"fn ecstore_implements_namespace_locking_contract()" \
"ECStore NamespaceLocking compile-time coverage test"
require_source_contains \
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
"fn ecstore_implements_heal_operations_contract()" \
"ECStore HealOperations compile-time coverage test"
require_source_contains \
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
"fn ecstore_implements_storage_namespace_locking_contract()" \
"ECStore storage-api NamespaceLocking compile-time coverage test"
require_source_contains \
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
"fn ecstore_implements_storage_object_io_contract()" \
"ECStore storage-api ObjectIO compile-time coverage test"
require_source_contains \
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
"fn ecstore_implements_storage_object_operations_contract()" \
"ECStore storage-api ObjectOperations compile-time coverage test"
require_source_contains \
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
"fn ecstore_implements_storage_list_operations_contract()" \
"ECStore storage-api ListOperations compile-time coverage test"
require_source_contains \
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
"fn ecstore_implements_storage_multipart_operations_contract()" \
"ECStore storage-api MultipartOperations compile-time coverage test"
require_source_contains \
"crates/ecstore/tests/ecstore_contract_compat_test.rs" \
"fn ecstore_implements_storage_heal_operations_contract()" \