mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
refactor: segment external storage contract imports (#3908)
This commit is contained in:
@@ -122,6 +122,16 @@ enum ReadRepairAdmissionOutcome {
|
||||
type ReadRepairAdmissionFuture = Pin<Box<dyn Future<Output = ReadRepairAdmissionOutcome> + Send>>;
|
||||
type ReadRepairAdmissionSubmitter = fn(rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture;
|
||||
|
||||
struct ReadRepairHealSubmission<'a> {
|
||||
bucket: &'a str,
|
||||
object: &'a str,
|
||||
version_id: Option<&'a str>,
|
||||
pool_index: usize,
|
||||
set_index: usize,
|
||||
part_number: Option<usize>,
|
||||
reason: &'static str,
|
||||
}
|
||||
|
||||
fn send_read_repair_heal_request(request: rustfs_common::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
|
||||
Box::pin(async {
|
||||
match send_heal_request_with_admission(request).await {
|
||||
@@ -141,6 +151,25 @@ async fn submit_read_repair_heal(
|
||||
reason: &'static str,
|
||||
) {
|
||||
submit_read_repair_heal_with_submitter(
|
||||
ReadRepairHealSubmission {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
pool_index,
|
||||
set_index,
|
||||
part_number,
|
||||
reason,
|
||||
},
|
||||
send_read_repair_heal_request,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn submit_read_repair_heal_with_submitter(
|
||||
submission: ReadRepairHealSubmission<'_>,
|
||||
submitter: ReadRepairAdmissionSubmitter,
|
||||
) {
|
||||
let ReadRepairHealSubmission {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
@@ -148,21 +177,8 @@ async fn submit_read_repair_heal(
|
||||
set_index,
|
||||
part_number,
|
||||
reason,
|
||||
send_read_repair_heal_request,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
} = submission;
|
||||
|
||||
async fn submit_read_repair_heal_with_submitter(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
pool_index: usize,
|
||||
set_index: usize,
|
||||
part_number: Option<usize>,
|
||||
reason: &'static str,
|
||||
submitter: ReadRepairAdmissionSubmitter,
|
||||
) {
|
||||
let Some(dedup_key) = reserve_read_repair_heal(bucket, object, version_id, pool_index, set_index).await else {
|
||||
record_read_repair_dedup("duplicate");
|
||||
debug!(
|
||||
@@ -1622,13 +1638,15 @@ mod metadata_cache_tests {
|
||||
let started = Instant::now();
|
||||
|
||||
submit_read_repair_heal_with_submitter(
|
||||
&bucket,
|
||||
"object",
|
||||
None,
|
||||
0,
|
||||
0,
|
||||
Some(1),
|
||||
"missing_shards",
|
||||
ReadRepairHealSubmission {
|
||||
bucket: &bucket,
|
||||
object: "object",
|
||||
version_id: None,
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
part_number: Some(1),
|
||||
reason: "missing_shards",
|
||||
},
|
||||
slow_read_repair_submitter,
|
||||
)
|
||||
.await;
|
||||
@@ -1654,13 +1672,15 @@ mod metadata_cache_tests {
|
||||
let bucket = format!("bucket-{}", Uuid::new_v4());
|
||||
|
||||
submit_read_repair_heal_with_submitter(
|
||||
&bucket,
|
||||
"object",
|
||||
None,
|
||||
0,
|
||||
0,
|
||||
Some(1),
|
||||
"missing_shards",
|
||||
ReadRepairHealSubmission {
|
||||
bucket: &bucket,
|
||||
object: "object",
|
||||
version_id: None,
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
part_number: Some(1),
|
||||
reason: "missing_shards",
|
||||
},
|
||||
dropped_read_repair_submitter,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -5,23 +5,19 @@ pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, DiskStore, STORA
|
||||
pub(crate) use rustfs_ecstore::api::erasure::Erasure;
|
||||
pub(crate) use rustfs_ecstore::api::object::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
pub(crate) use rustfs_ecstore::api::{error::Error, storage::ECStore};
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
CompletePart, DeletedObject, HTTPRangeSpec, HealOperations as StorageHealOperations, ListMultipartsInfo,
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
ListOperations as StorageListOperations, ListPartsInfo, MultipartInfo, MultipartOperations as StorageMultipartOperations,
|
||||
MultipartUploadResult, NamespaceLocking as StorageNamespaceLocking, ObjectIO as StorageObjectIO,
|
||||
ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations as StorageObjectOperations, ObjectToDelete, PartInfo,
|
||||
StorageAdminApi, WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) mod contract_compat {
|
||||
pub(crate) use super::{
|
||||
CompletePart, DeletedObject, DiskStore, ECStore, Error, GetObjectReader, HTTPRangeSpec, ListMultipartsInfo,
|
||||
ListPartsInfo, MultipartInfo, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader,
|
||||
StorageAdminApi, StorageHealOperations, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageListOperations,
|
||||
StorageMultipartOperations, StorageNamespaceLocking, StorageObjectIO, StorageObjectInfoOrErr, StorageObjectOperations,
|
||||
StorageWalkOptions,
|
||||
pub(crate) use super::storage_contracts::{
|
||||
CompletePart, DeletedObject, HTTPRangeSpec, HealOperations as StorageHealOperations, ListMultipartsInfo,
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
ListOperations as StorageListOperations, ListPartsInfo, MultipartInfo, MultipartOperations as StorageMultipartOperations,
|
||||
MultipartUploadResult, NamespaceLocking as StorageNamespaceLocking, ObjectIO as StorageObjectIO,
|
||||
ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations as StorageObjectOperations, ObjectToDelete, PartInfo,
|
||||
StorageAdminApi, WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
|
||||
pub(crate) use super::{DiskStore, ECStore, Error, GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
}
|
||||
|
||||
pub(crate) mod legacy_bitrot_read {
|
||||
|
||||
@@ -24,15 +24,15 @@ pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_
|
||||
pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
|
||||
pub(crate) use rustfs_ecstore::api::global::GLOBAL_LOCAL_DISK_MAP as ECSTORE_GLOBAL_LOCAL_DISK_MAP;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
BucketInfo, BucketOperations, DiskSetSelector, HealOperations, ListOperations, ObjectIO, ObjectOperations, StorageAdminApi,
|
||||
};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) mod owner {
|
||||
pub(crate) use super::storage_contracts::{ObjectIO, ObjectOperations};
|
||||
|
||||
pub(crate) use super::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_GLOBAL_LOCAL_DISK_MAP, ECSTORE_RUSTFS_META_BUCKET,
|
||||
EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore,
|
||||
EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations,
|
||||
EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -40,7 +40,7 @@ pub(crate) mod owner {
|
||||
}
|
||||
|
||||
pub(crate) mod storage {
|
||||
pub(crate) use super::{
|
||||
pub(crate) use super::storage_contracts::{
|
||||
BucketInfo, BucketOperations, DiskSetSelector, HealOperations, ListOperations, ObjectIO, ObjectOperations,
|
||||
StorageAdminApi,
|
||||
};
|
||||
@@ -48,5 +48,5 @@ pub(crate) mod storage {
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod status {
|
||||
pub(crate) use super::BucketInfo;
|
||||
pub(crate) use super::storage_contracts::BucketInfo;
|
||||
}
|
||||
|
||||
@@ -13,8 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint};
|
||||
pub(crate) use rustfs_storage_api::BucketInfo;
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) mod bug_fixes {
|
||||
pub(crate) use super::{BucketInfo, DiskStore, Endpoint};
|
||||
pub(crate) use super::storage_contracts::BucketInfo;
|
||||
pub(crate) use super::{DiskStore, Endpoint};
|
||||
}
|
||||
|
||||
@@ -16,11 +16,12 @@ pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_s
|
||||
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
|
||||
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
|
||||
pub(crate) use rustfs_storage_api::{BucketOperations, BucketOptions, ObjectIO, ObjectOperations};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) mod integration {
|
||||
pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, ObjectIO, ObjectOperations};
|
||||
|
||||
pub(crate) use super::{
|
||||
BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, ObjectIO, ObjectOperations,
|
||||
PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
|
||||
ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -29,14 +29,14 @@ use rustfs_ecstore::api::notification::{
|
||||
NotificationPeerErr as EcstoreNotificationPeerErr, NotificationSys as EcstoreNotificationSys, get_global_notification_sys,
|
||||
};
|
||||
use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
pub(crate) use rustfs_storage_api::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) type IamEcstoreError = EcstoreErrorType;
|
||||
pub(crate) type IamStorageError = EcstoreStorageError;
|
||||
pub(crate) type IamStorageResult<T> = EcstoreResultType<T>;
|
||||
pub(crate) type IamStore = EcstoreStore;
|
||||
pub(crate) type IamConfigObjectInfo = <IamStore as ObjectOperations>::ObjectInfo;
|
||||
pub(crate) type IamConfigObjectOptions = <IamStore as ObjectOperations>::ObjectOptions;
|
||||
pub(crate) type IamConfigObjectInfo = <IamStore as storage_contracts::ObjectOperations>::ObjectInfo;
|
||||
pub(crate) type IamConfigObjectOptions = <IamStore as storage_contracts::ObjectOperations>::ObjectOptions;
|
||||
pub(crate) type IamNotificationSys = EcstoreNotificationSys;
|
||||
pub(crate) type IamEcstoreNotificationPeerErr = EcstoreNotificationPeerErr;
|
||||
|
||||
@@ -90,7 +90,7 @@ pub(crate) mod crate_boundary {
|
||||
}
|
||||
|
||||
pub(crate) mod object_store {
|
||||
pub(crate) use super::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
|
||||
pub(crate) use super::storage_contracts::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
|
||||
}
|
||||
|
||||
pub(crate) mod runtime {
|
||||
|
||||
@@ -30,13 +30,15 @@ pub(crate) use rustfs_ecstore::api::global::{
|
||||
get_global_bucket_monitor as obs_get_global_bucket_monitor, resolve_object_store_handle as obs_resolve_object_store_handle,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as ObsStore;
|
||||
pub(crate) use rustfs_storage_api::{BucketOperations, BucketOptions, StorageAdminApi};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) mod metrics {
|
||||
pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, StorageAdminApi};
|
||||
|
||||
pub(crate) use super::{
|
||||
BucketOperations, BucketOptions, OBS_GLOBAL_EXPIRY_STATE, OBS_GLOBAL_REPLICATION_STATS, OBS_GLOBAL_TRANSITION_STATE,
|
||||
ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsReplicationStats, ObsStore, StorageAdminApi,
|
||||
obs_get_global_bucket_monitor, obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free,
|
||||
obs_load_data_usage_from_backend, obs_resolve_object_store_handle,
|
||||
OBS_GLOBAL_EXPIRY_STATE, OBS_GLOBAL_REPLICATION_STATS, OBS_GLOBAL_TRANSITION_STATE, ObsBucketBandwidthMonitor,
|
||||
ObsEcstoreResult, ObsReplicationStats, ObsStore, obs_get_global_bucket_monitor, obs_get_quota_config,
|
||||
obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_load_data_usage_from_backend,
|
||||
obs_resolve_object_store_handle,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -21,25 +21,24 @@ use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
pub(crate) use rustfs_ecstore::api::error::Result as SwiftStorageResult;
|
||||
pub(crate) use rustfs_ecstore::api::global::resolve_object_store_handle as resolve_swift_object_store_handle;
|
||||
use rustfs_ecstore::api::storage::ECStore as SwiftStore;
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, HTTPRangeSpec, ListOperations, MakeBucketOptions, ObjectIO,
|
||||
ObjectOperations,
|
||||
};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) mod account {
|
||||
pub(crate) use super::{BucketOperations, MakeBucketOptions};
|
||||
pub(crate) use super::storage_contracts::{BucketOperations, MakeBucketOptions};
|
||||
}
|
||||
|
||||
pub(crate) mod container {
|
||||
pub(crate) use super::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListOperations, MakeBucketOptions};
|
||||
pub(crate) use super::storage_contracts::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListOperations, MakeBucketOptions,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod large_object {
|
||||
pub(crate) use super::HTTPRangeSpec;
|
||||
pub(crate) use super::storage_contracts::HTTPRangeSpec;
|
||||
}
|
||||
|
||||
pub(crate) mod object {
|
||||
pub(crate) use super::{BucketOperations, BucketOptions, HTTPRangeSpec, ObjectIO, ObjectOperations};
|
||||
pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, HTTPRangeSpec, ObjectIO, ObjectOperations};
|
||||
}
|
||||
|
||||
pub(crate) mod public_api {
|
||||
@@ -48,13 +47,13 @@ pub(crate) mod public_api {
|
||||
}
|
||||
|
||||
pub(crate) mod versioning {
|
||||
pub(crate) use super::{ListOperations, ObjectOperations};
|
||||
pub(crate) use super::storage_contracts::{ListOperations, ObjectOperations};
|
||||
}
|
||||
|
||||
pub type SwiftGetObjectReader = <SwiftStore as ObjectIO>::GetObjectReader;
|
||||
pub type SwiftObjectInfo = <SwiftStore as ObjectOperations>::ObjectInfo;
|
||||
pub type SwiftObjectOptions = <SwiftStore as ObjectOperations>::ObjectOptions;
|
||||
pub type SwiftPutObjReader = <SwiftStore as ObjectIO>::PutObjectReader;
|
||||
pub type SwiftGetObjectReader = <SwiftStore as storage_contracts::ObjectIO>::GetObjectReader;
|
||||
pub type SwiftObjectInfo = <SwiftStore as storage_contracts::ObjectOperations>::ObjectInfo;
|
||||
pub type SwiftObjectOptions = <SwiftStore as storage_contracts::ObjectOperations>::ObjectOptions;
|
||||
pub type SwiftPutObjReader = <SwiftStore as storage_contracts::ObjectIO>::PutObjectReader;
|
||||
|
||||
pub(crate) async fn get_swift_bucket_metadata(bucket: &str) -> SwiftStorageResult<Arc<SwiftBucketMetadata>> {
|
||||
get_swift_bucket_metadata_from_backend(bucket).await
|
||||
|
||||
@@ -23,10 +23,10 @@ use rustfs_ecstore::api::error::{
|
||||
use rustfs_ecstore::api::global::resolve_object_store_handle as resolve_select_object_store_handle_from_backend;
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::DEFAULT_READ_BUFFER_SIZE as SELECT_DEFAULT_READ_BUFFER_SIZE;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as SelectStore;
|
||||
pub(crate) use rustfs_storage_api::{HTTPRangeSpec, ObjectIO, ObjectOperations};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) mod object_store {
|
||||
pub(crate) use super::{HTTPRangeSpec, ObjectIO, ObjectOperations};
|
||||
pub(crate) use super::storage_contracts::{HTTPRangeSpec, ObjectIO, ObjectOperations};
|
||||
}
|
||||
|
||||
pub(crate) mod crate_boundary {
|
||||
@@ -37,9 +37,9 @@ pub(crate) mod crate_boundary {
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) type SelectGetObjectReader = <SelectStore as ObjectIO>::GetObjectReader;
|
||||
pub(crate) type SelectObjectInfo = <SelectStore as ObjectOperations>::ObjectInfo;
|
||||
pub(crate) type SelectObjectOptions = <SelectStore as ObjectOperations>::ObjectOptions;
|
||||
pub(crate) type SelectGetObjectReader = <SelectStore as storage_contracts::ObjectIO>::GetObjectReader;
|
||||
pub(crate) type SelectObjectInfo = <SelectStore as storage_contracts::ObjectOperations>::ObjectInfo;
|
||||
pub(crate) type SelectObjectOptions = <SelectStore as storage_contracts::ObjectOperations>::ObjectOptions;
|
||||
|
||||
pub(crate) fn resolve_select_object_store_handle() -> Option<Arc<SelectStore>> {
|
||||
resolve_select_object_store_handle_from_backend()
|
||||
|
||||
@@ -72,12 +72,11 @@ pub(crate) use rustfs_ecstore::api::global::{
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
pub(crate) use rustfs_ecstore::api::tier::tier_config::TierConfig as EcstoreTierConfig;
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, HTTPRangeSpec, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
ObjectToDelete, StorageAdminApi,
|
||||
};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) mod owner {
|
||||
pub(crate) use super::storage_contracts::{HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete};
|
||||
|
||||
pub(crate) use super::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_RUSTFS_META_BUCKET, ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS,
|
||||
ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys,
|
||||
@@ -86,12 +85,12 @@ pub(crate) mod owner {
|
||||
EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreObjectOpts, EcstoreReplicationConfig,
|
||||
EcstoreReplicationConfigurationExt, EcstoreReplicationHealQueueResult, EcstoreReplicationQueueAdmission,
|
||||
EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig,
|
||||
EcstoreVersioningApi, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete, ecstore_apply_expiry_rule,
|
||||
ecstore_apply_transition_rule, ecstore_get_global_expiry_state, ecstore_get_global_tier_config_mgr,
|
||||
ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure,
|
||||
ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_path2_bucket_object,
|
||||
ecstore_path2_bucket_object_with_base_path, ecstore_queue_replication_heal_internal, ecstore_read_config,
|
||||
ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
|
||||
EcstoreVersioningApi, ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_get_global_expiry_state,
|
||||
ecstore_get_global_tier_config_mgr, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
|
||||
ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket,
|
||||
ecstore_list_path_raw, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
|
||||
ecstore_queue_replication_heal_internal, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info,
|
||||
ecstore_resolve_object_store_handle, ecstore_save_config,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -99,11 +98,11 @@ pub(crate) mod owner {
|
||||
}
|
||||
|
||||
pub(crate) mod scan {
|
||||
pub(crate) use super::{BucketOperations, BucketOptions, NamespaceLocking};
|
||||
pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, NamespaceLocking};
|
||||
}
|
||||
|
||||
pub(crate) mod scanner_io {
|
||||
pub(crate) use super::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
|
||||
pub(crate) use super::storage_contracts::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::{HTTPRangeSpec, ObjectIO};
|
||||
pub(crate) use super::storage_contracts::{HTTPRangeSpec, ObjectIO};
|
||||
}
|
||||
|
||||
@@ -31,17 +31,18 @@ pub(crate) use rustfs_ecstore::api::tier::tier_config::{TierConfig, TierMinIO, T
|
||||
pub(crate) use rustfs_ecstore::api::tier::warm_backend::{
|
||||
WarmBackend as ScannerWarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
||||
};
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
BucketOperations, BucketOptions, CompletePart, ListOperations, MakeBucketOptions, MultipartOperations, ObjectIO,
|
||||
ObjectOperations,
|
||||
};
|
||||
use rustfs_storage_api as storage_contracts;
|
||||
|
||||
pub(crate) mod lifecycle {
|
||||
pub(crate) use super::storage_contracts::{
|
||||
BucketOperations, BucketOptions, CompletePart, ListOperations, MakeBucketOptions, MultipartOperations, ObjectIO,
|
||||
ObjectOperations,
|
||||
};
|
||||
|
||||
pub(crate) use super::{
|
||||
BUCKET_LIFECYCLE_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, CompletePart, DiskAPI, DiskOption,
|
||||
ECStore, Endpoint, EndpointServerPools, Endpoints, ListOperations, MakeBucketOptions, MultipartOperations, ObjectIO,
|
||||
ObjectOperations, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, ScannerWarmBackend, TierConfig, TierMinIO,
|
||||
TierType, TransitionOptions, WarmBackendGetOpts, build_transition_put_options, enqueue_transition_for_existing_objects,
|
||||
BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DiskAPI, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
|
||||
PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, ScannerWarmBackend, TierConfig, TierMinIO, TierType,
|
||||
TransitionOptions, WarmBackendGetOpts, build_transition_put_options, enqueue_transition_for_existing_objects,
|
||||
get_bucket_metadata, get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks,
|
||||
new_disk, path2_bucket_object_with_base_path, update_bucket_metadata,
|
||||
};
|
||||
|
||||
@@ -5,14 +5,13 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
## Current Context
|
||||
|
||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||
- Branch: `overtrue/arch-residual-storage-api-domain-boundaries`
|
||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221/API-222/API-223/API-224/API-225/API-226/API-227/API-228/API-229/API-230/API-231/API-232/API-233/API-234/API-235/API-236/API-237/API-238/API-239/API-240/CTX-002`.
|
||||
- Based on: rebased onto current `origin/main` after PR #3897 and PR #3900
|
||||
merged on top of PR #3903.
|
||||
- Branch: `overtrue/arch-storage-api-residual-flat-consumers`
|
||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221/API-222/API-223/API-224/API-225/API-226/API-227/API-228/API-229/API-230/API-231/API-232/API-233/API-234/API-235/API-236/API-237/API-238/API-239/API-240/API-241/CTX-002`.
|
||||
- Based on: rebased onto current `origin/main` after PR #3905 merged.
|
||||
- PR type for this branch: `consumer-migration`
|
||||
- Runtime behavior changes: none expected for API-241; residual notify,
|
||||
ECStore test/bench, admin config, and storage RPC consumers still use the
|
||||
same owner symbols through narrower local boundaries and aliases.
|
||||
- Runtime behavior changes: none expected for API-242; external/test local
|
||||
storage API boundaries still expose the same storage-api contracts through
|
||||
consumer-domain modules.
|
||||
- Rust code changes: route replication pool, outbound TLS generation, runtime
|
||||
region, KMS encryption service, runtime support handles, S3 Select DB,
|
||||
internode RPC metrics, IAM authorization/handler reads, notification
|
||||
@@ -81,7 +80,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
storage API consumers through external crate-local domain modules, plus
|
||||
residual notify crate-boundary consumers, ECStore test/bench storage API
|
||||
domain modules, and local ECStore type aliases for admin config and storage
|
||||
RPC paths.
|
||||
RPC paths, plus external/test local storage API contract imports through
|
||||
local `storage_contracts` aliases and consumer-domain modules instead of
|
||||
root re-exports.
|
||||
- CI/script changes: lock completed owner and test/fuzz boundaries against
|
||||
bare/glob imports, scattered raw ECStore facade subpaths, and startup
|
||||
runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade
|
||||
@@ -91,12 +92,13 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
event-bridge thin module regressions, plus IAM runtime-source bypasses;
|
||||
accept the reviewed AppContext resolver reverse dependencies in the layer
|
||||
baseline, and block direct admin AppContext resolver consumers outside the
|
||||
admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, reject root/server/startup direct storage facade regressions from migrated outer consumers, reject root/server/startup direct storage contract imports from migrated outer consumers, reject app/admin direct storage contract imports from migrated owner consumers, keep app S3 helper imports routed through `app::storage_api`, reject scanner/heal direct ECStore or storage contract imports outside their local `storage_api` boundaries, reject external runtime/test/fuzz ECStore or storage contract imports outside their local `storage_api` boundaries, reject storage owner direct ECStore/storage-api imports outside the owner-local `storage_api` boundary, reject ECStore internal direct storage-api imports outside the owner-local `storage_api_contracts` boundary, reject direct storage ECFS app usecase construction outside the storage S3 API boundary, reject flat root `storage_api` imports outside the new root-local domain modules, reject flat admin `storage_api` imports outside the new admin domain modules, reject flat app `storage_api` imports outside the new app consumer-domain modules, reject flat external crate-local `storage_api` imports outside the new consumer-domain modules, and reject residual flat local `storage_api` imports from notify, ECStore tests/benches, and remaining RustFS app storage API type references.
|
||||
- Docs changes: record the API-136 through API-241 owner facade,
|
||||
admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, reject app usecase bypasses for migrated storage IO/compression/set-disk helpers, reject app usecase/test bypasses for migrated storage error, ETag, and storage-class helpers, reject app root bucket owner facade bypasses from migrated app consumers, reject app/admin runtime/data-usage root facade regressions, reject admin root storage facade regressions from migrated admin consumers, reject root/server/startup direct storage facade regressions from migrated outer consumers, reject root/server/startup direct storage contract imports from migrated outer consumers, reject app/admin direct storage contract imports from migrated owner consumers, keep app S3 helper imports routed through `app::storage_api`, reject scanner/heal direct ECStore or storage contract imports outside their local `storage_api` boundaries, reject external runtime/test/fuzz ECStore or storage contract imports outside their local `storage_api` boundaries, reject storage owner direct ECStore/storage-api imports outside the owner-local `storage_api` boundary, reject ECStore internal direct storage-api imports outside the owner-local `storage_api_contracts` boundary, reject direct storage ECFS app usecase construction outside the storage S3 API boundary, reject flat root `storage_api` imports outside the new root-local domain modules, reject flat admin `storage_api` imports outside the new admin domain modules, reject flat app `storage_api` imports outside the new app consumer-domain modules, reject flat external crate-local `storage_api` imports outside the new consumer-domain modules, reject residual flat local `storage_api` imports from notify, ECStore tests/benches, and remaining RustFS app storage API type references, and reject external/test local storage API root contract re-exports.
|
||||
- Docs changes: record the API-136 through API-242 owner facade,
|
||||
runtime-source, ECFS usecase, root storage API domain-boundary, and admin
|
||||
storage API domain-boundary cleanup, and app storage API consumer-domain
|
||||
cleanup, plus external crate-local and residual local storage API
|
||||
consumer-domain cleanup.
|
||||
consumer-domain cleanup, plus external/test storage contract root re-export
|
||||
cleanup.
|
||||
|
||||
## Phase 0 Tasks
|
||||
|
||||
@@ -5547,14 +5549,32 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
and layer guards, diff hygiene, residual flat local storage-api scan, Rust
|
||||
risk scan, and full PR gate before PR.
|
||||
|
||||
- [x] `API-242` Segment external storage API contract imports by domain module.
|
||||
- Do: replace external/test local storage API root contract re-exports with
|
||||
local `storage_contracts` aliases and consumer-domain module exports.
|
||||
- Acceptance: ECStore tests, heal source/tests, IAM, OBS metrics, Swift, S3
|
||||
Select, and scanner source/tests no longer expose storage-api contracts from
|
||||
their local `storage_api` roots; migration rules reject root re-export
|
||||
regressions.
|
||||
- Must preserve: ECStore compatibility/read fixtures, heal storage/status
|
||||
behavior, IAM object-store operations, OBS metrics collection, Swift object
|
||||
and bucket behavior, S3 Select object reads, scanner lifecycle/tier
|
||||
behavior, and test harness setup.
|
||||
- Verification: focused external/test crate compile, formatting, migration
|
||||
and layer guards, diff hygiene, external root storage-api re-export scan,
|
||||
Rust risk scan, and full PR gate before PR.
|
||||
|
||||
## Next PRs
|
||||
|
||||
1. `consumer-migration`: continue larger owner boundary batches after API-241.
|
||||
1. `consumer-migration`: continue larger owner boundary batches after API-242.
|
||||
|
||||
## Pre-Push Review Log
|
||||
|
||||
| Expert | Status | Notes |
|
||||
|---|---|---|
|
||||
| Quality/architecture | pass | API-242 removes external/test local storage API root contract re-exports and keeps contract exposure inside consumer-domain modules. |
|
||||
| Migration preservation | pass | ECStore tests, heal, IAM, OBS metrics, Swift, S3 Select, and scanner consumers keep the same storage-api contracts and call paths. |
|
||||
| Testing/verification | pass | Focused external/test compile, formatting, migration/layer guards, external root storage-api re-export scan, diff hygiene, diff-added Rust risk scan, and full PR gate passed before PR. |
|
||||
| Quality/architecture | pass | API-241 segments residual local storage API imports into notify, ECStore test/bench, and RustFS local type boundaries. |
|
||||
| Migration preservation | pass | Notify config, ECStore contract/bitrot/read fixtures, benchmark setup, admin config, and storage RPC consumers keep the same owner symbols and call paths. |
|
||||
| Testing/verification | pass | Focused compile, formatting, migration/layer guards, residual flat local storage-api scan, diff hygiene, diff-added Rust risk scan, and full PR gate passed before PR. |
|
||||
@@ -5819,6 +5839,25 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
|
||||
Passed before push:
|
||||
|
||||
- Issue #660 API-242 current slice:
|
||||
- Branch freshness check: rebased onto current `origin/main` after PR #3905
|
||||
merged.
|
||||
- `cargo check --tests --benches -p rustfs-scanner -p rustfs-heal -p rustfs-iam -p rustfs-obs -p rustfs-s3select-api -p rustfs-protocols -p rustfs-ecstore -p e2e_test -F rustfs-protocols/swift`:
|
||||
passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||
- External storage-api root re-export scan: passed; ECStore tests, heal
|
||||
source/tests, IAM, OBS metrics, Swift, S3 Select, and scanner source/tests
|
||||
now expose storage-api contracts from consumer-domain modules through local
|
||||
`storage_contracts` aliases.
|
||||
- Diff-added Rust risk scan: passed; no new production unwrap/expect,
|
||||
numeric cast, String error, Box dyn Error, print macro, or relaxed atomic
|
||||
ordering lines.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `make pre-pr`: passed.
|
||||
|
||||
- Issue #660 API-241 current slice:
|
||||
- Branch freshness check: rebased onto current `origin/main` after PR #3897
|
||||
and PR #3900 merged on top of PR #3903.
|
||||
|
||||
@@ -119,6 +119,7 @@ RUSTFS_ADMIN_ECSTORE_SOURCE_HITS_FILE="${TMP_DIR}/rustfs_admin_ecstore_source_hi
|
||||
EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/external_runtime_ecstore_compat_bypass_hits.txt"
|
||||
EXTERNAL_RUNTIME_STORAGE_API_BYPASS_HITS_FILE="${TMP_DIR}/external_runtime_storage_api_bypass_hits.txt"
|
||||
EXTERNAL_STORAGE_API_DOMAIN_BYPASS_HITS_FILE="${TMP_DIR}/external_storage_api_domain_bypass_hits.txt"
|
||||
EXTERNAL_STORAGE_API_ROOT_REEXPORT_HITS_FILE="${TMP_DIR}/external_storage_api_root_reexport_hits.txt"
|
||||
EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/external_test_ecstore_compat_bypass_hits.txt"
|
||||
FUZZ_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/fuzz_ecstore_compat_bypass_hits.txt"
|
||||
ALL_STORAGE_COMPAT_SELF_FACADE_PATH_HITS_FILE="${TMP_DIR}/all_storage_compat_self_facade_path_hits.txt"
|
||||
@@ -1441,6 +1442,28 @@ if [[ -s "$EXTERNAL_STORAGE_API_DOMAIN_BYPASS_HITS_FILE" ]]; then
|
||||
report_failure "external local storage_api consumers must use domain modules instead of flat imports: $(paste -sd '; ' "$EXTERNAL_STORAGE_API_DOMAIN_BYPASS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename '^pub(?:\(crate\))? use rustfs_storage_api' \
|
||||
crates/ecstore/benches \
|
||||
crates/ecstore/tests \
|
||||
crates/heal/src/heal \
|
||||
crates/heal/tests \
|
||||
crates/iam/src \
|
||||
crates/obs/src/metrics \
|
||||
crates/protocols/src/swift \
|
||||
crates/s3select-api/src \
|
||||
crates/scanner/src \
|
||||
crates/scanner/tests \
|
||||
crates/e2e_test/src \
|
||||
--glob '**/storage_api.rs' \
|
||||
--glob '**/storage_api/mod.rs' || true
|
||||
) >"$EXTERNAL_STORAGE_API_ROOT_REEXPORT_HITS_FILE"
|
||||
|
||||
if [[ -s "$EXTERNAL_STORAGE_API_ROOT_REEXPORT_HITS_FILE" ]]; then
|
||||
report_failure "external local storage_api boundaries must expose storage-api contracts from consumer-domain modules, not root re-exports: $(paste -sd '; ' "$EXTERNAL_STORAGE_API_ROOT_REEXPORT_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'get_global_(?:action_cred|server_config|notification_sys)' \
|
||||
|
||||
Reference in New Issue
Block a user