mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-19 19:16:17 +00:00
refactor: segment external storage api boundaries (#3903)
This commit is contained in:
@@ -24,7 +24,7 @@ use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, Generall
|
||||
use tonic::Request;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::storage_api::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
use crate::storage_api::grpc_lock::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
|
||||
/// gRPC lock client without authentication for testing
|
||||
/// Similar to RemoteClient but uses no_auth client
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::workspace_root;
|
||||
use crate::storage_api::{
|
||||
use crate::storage_api::node_interact::{
|
||||
TonicInterceptor, VolumeInfo, WalkDirOptions, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
};
|
||||
use futures::future::join_all;
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::common::{
|
||||
RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
|
||||
rustfs_binary_path,
|
||||
};
|
||||
use crate::storage_api::BucketTargetSys;
|
||||
use crate::storage_api::replication_extension::BucketTargetSys;
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
|
||||
@@ -19,3 +19,19 @@ pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod node_interact {
|
||||
pub(crate) use super::{
|
||||
TonicInterceptor, VolumeInfo, WalkDirOptions, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod grpc_lock {
|
||||
pub(crate) use super::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod replication_extension {
|
||||
pub(crate) use super::BucketTargetSys;
|
||||
}
|
||||
|
||||
@@ -563,13 +563,13 @@ mod tests {
|
||||
async fn format_disk(&self, _endpoint: &Endpoint) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn get_bucket_info(&self, _bucket: &str) -> crate::Result<Option<crate::heal::storage_api::BucketInfo>> {
|
||||
async fn get_bucket_info(&self, _bucket: &str) -> crate::Result<Option<crate::heal::storage_api::status::BucketInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn heal_bucket_metadata(&self, _bucket: &str) -> crate::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn list_buckets(&self) -> crate::Result<Vec<crate::heal::storage_api::BucketInfo>> {
|
||||
async fn list_buckets(&self) -> crate::Result<Vec<crate::heal::storage_api::status::BucketInfo>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
async fn object_exists(&self, _bucket: &str, _object: &str) -> crate::Result<bool> {
|
||||
|
||||
@@ -2408,7 +2408,7 @@ mod tests {
|
||||
use rustfs_common::heal_channel::{HealOpts, HealRequestSource};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
|
||||
use super::super::{DiskStore, Endpoint, storage_api::BucketInfo};
|
||||
use super::super::{DiskStore, Endpoint, storage_api::status::BucketInfo};
|
||||
|
||||
struct MockStorage;
|
||||
|
||||
|
||||
@@ -23,13 +23,13 @@ pub(crate) mod storage_api;
|
||||
pub mod task;
|
||||
pub mod utils;
|
||||
|
||||
use storage_api::{
|
||||
use storage_api::owner::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_GLOBAL_LOCAL_DISK_MAP, ECSTORE_RUSTFS_META_BUCKET,
|
||||
EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore,
|
||||
EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use storage_api::{EcstoreDiskOption, ecstore_new_disk};
|
||||
use storage_api::owner::{EcstoreDiskOption, ecstore_new_disk};
|
||||
|
||||
pub use erasure_healer::ErasureSetHealer;
|
||||
pub use manager::{HealManager, HealOperationsSnapshot, HealPriorityCounts, HealSourceCounts};
|
||||
|
||||
@@ -19,7 +19,7 @@ use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use super::storage_api::{
|
||||
use super::storage_api::storage::{
|
||||
BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _,
|
||||
ObjectOperations as _, StorageAdminApi,
|
||||
};
|
||||
|
||||
@@ -27,3 +27,26 @@ pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
BucketInfo, BucketOperations, DiskSetSelector, HealOperations, ListOperations, ObjectIO, ObjectOperations, StorageAdminApi,
|
||||
};
|
||||
|
||||
pub(crate) mod owner {
|
||||
pub(crate) use super::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_GLOBAL_LOCAL_DISK_MAP, ECSTORE_RUSTFS_META_BUCKET,
|
||||
EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore,
|
||||
EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore, ObjectIO, ObjectOperations,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::{EcstoreDiskOption, ecstore_new_disk};
|
||||
}
|
||||
|
||||
pub(crate) mod storage {
|
||||
pub(crate) use super::{
|
||||
BucketInfo, BucketOperations, DiskSetSelector, HealOperations, ListOperations, ObjectIO, ObjectOperations,
|
||||
StorageAdminApi,
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod status {
|
||||
pub(crate) use super::BucketInfo;
|
||||
}
|
||||
|
||||
@@ -2208,7 +2208,7 @@ mod tests {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use super::super::storage_api::BucketInfo;
|
||||
use super::super::storage_api::status::BucketInfo;
|
||||
#[derive(Default)]
|
||||
struct MockStorage {
|
||||
listed: Mutex<bool>,
|
||||
|
||||
@@ -21,7 +21,7 @@ use tokio_util::sync::CancellationToken;
|
||||
#[path = "endpoint_index_test/storage_api.rs"]
|
||||
mod storage_api;
|
||||
|
||||
use storage_api::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks};
|
||||
use storage_api::endpoint_index::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks};
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_endpoint_index_settings() -> anyhow::Result<()> {
|
||||
|
||||
@@ -15,3 +15,7 @@
|
||||
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
|
||||
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
|
||||
|
||||
pub(crate) mod endpoint_index {
|
||||
pub(crate) use super::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks};
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ use rustfs_heal::heal::{
|
||||
#[path = "heal_bug_fixes_test/storage_api.rs"]
|
||||
mod storage_api;
|
||||
|
||||
use storage_api::{BucketInfo, DiskStore, Endpoint};
|
||||
use storage_api::bug_fixes::{BucketInfo, DiskStore, Endpoint};
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_to_heal_request_no_panic() {
|
||||
|
||||
@@ -14,3 +14,7 @@
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint};
|
||||
pub(crate) use rustfs_storage_api::BucketInfo;
|
||||
|
||||
pub(crate) mod bug_fixes {
|
||||
pub(crate) use super::{BucketInfo, DiskStore, Endpoint};
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ use walkdir::WalkDir;
|
||||
#[path = "heal_integration_test/storage_api.rs"]
|
||||
mod storage_api;
|
||||
|
||||
use storage_api::{
|
||||
use storage_api::integration::{
|
||||
BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, ObjectIO as _, ObjectOperations as _,
|
||||
PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
|
||||
};
|
||||
|
||||
@@ -17,3 +17,10 @@ pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
|
||||
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
|
||||
pub(crate) use rustfs_storage_api::{BucketOperations, BucketOptions, ObjectIO, ObjectOperations};
|
||||
|
||||
pub(crate) mod integration {
|
||||
pub(crate) use super::{
|
||||
BucketOperations, BucketOptions, ECStore, Endpoint, EndpointServerPools, Endpoints, ObjectIO, ObjectOperations,
|
||||
PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ mod storage_api;
|
||||
pub mod store;
|
||||
pub mod sys;
|
||||
pub mod utils;
|
||||
pub(crate) use storage_api::{
|
||||
pub(crate) use storage_api::crate_boundary::{
|
||||
IAM_CONFIG_ROOT_PREFIX, IamEcstoreError, IamStorageError, IamStore, classify_iam_system_path_failure_reason,
|
||||
delete_iam_config, is_iam_first_cluster_node_local, read_iam_config_no_lock, read_iam_config_with_metadata, save_iam_config,
|
||||
save_iam_config_with_opts,
|
||||
@@ -53,8 +53,8 @@ pub(crate) struct IamNotificationPeerErr {
|
||||
pub(crate) err: Option<IamEcstoreError>,
|
||||
}
|
||||
|
||||
impl From<storage_api::IamEcstoreNotificationPeerErr> for IamNotificationPeerErr {
|
||||
fn from(value: storage_api::IamEcstoreNotificationPeerErr) -> Self {
|
||||
impl From<storage_api::crate_boundary::IamEcstoreNotificationPeerErr> for IamNotificationPeerErr {
|
||||
fn from(value: storage_api::crate_boundary::IamEcstoreNotificationPeerErr) -> Self {
|
||||
Self { err: value.err }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage_api::{IamNotificationSys, notification_sys as ecstore_notification_sys};
|
||||
use crate::storage_api::runtime::{IamNotificationSys, notification_sys as ecstore_notification_sys};
|
||||
use rustfs_config::server_config::{Config as ServerConfig, get_global_server_config};
|
||||
use rustfs_credentials::{Credentials, get_global_action_cred};
|
||||
|
||||
|
||||
@@ -80,3 +80,19 @@ pub(crate) async fn is_iam_first_cluster_node_local() -> bool {
|
||||
pub(crate) fn notification_sys() -> Option<&'static IamNotificationSys> {
|
||||
get_global_notification_sys()
|
||||
}
|
||||
|
||||
pub(crate) mod crate_boundary {
|
||||
pub(crate) use super::{
|
||||
IAM_CONFIG_ROOT_PREFIX, IamEcstoreError, IamEcstoreNotificationPeerErr, IamStorageError, IamStore,
|
||||
classify_iam_system_path_failure_reason, delete_iam_config, is_iam_first_cluster_node_local, read_iam_config_no_lock,
|
||||
read_iam_config_with_metadata, save_iam_config, save_iam_config_with_opts,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod object_store {
|
||||
pub(crate) use super::{HTTPPreconditions, ListOperations, ObjectInfoOrErr, ObjectOperations};
|
||||
}
|
||||
|
||||
pub(crate) mod runtime {
|
||||
pub(crate) use super::{IamNotificationSys, notification_sys};
|
||||
}
|
||||
|
||||
@@ -37,7 +37,9 @@ use tokio::sync::mpsc::{self, Sender};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use crate::storage_api::{HTTPPreconditions, ListOperations as _, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations};
|
||||
use crate::storage_api::object_store::{
|
||||
HTTPPreconditions, ListOperations as _, ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations,
|
||||
};
|
||||
|
||||
pub static IAM_CONFIG_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{IAM_CONFIG_ROOT_PREFIX}/iam"));
|
||||
pub static IAM_CONFIG_USERS_PREFIX: LazyLock<String> = LazyLock::new(|| format!("{IAM_CONFIG_ROOT_PREFIX}/iam/users/"));
|
||||
|
||||
@@ -30,7 +30,7 @@ pub use scheduler::{
|
||||
MetricsRuntimeShutdownHandle, MetricsRuntimeStatusSnapshot, MetricsRuntimeWorkerMutation, init_metrics_collectors,
|
||||
init_metrics_runtime, metrics_runtime_controller_snapshot, metrics_runtime_status_snapshot,
|
||||
};
|
||||
pub(crate) use storage_api::{
|
||||
pub(crate) use storage_api::metrics::{
|
||||
BucketOperations, BucketOptions, OBS_GLOBAL_EXPIRY_STATE, OBS_GLOBAL_REPLICATION_STATS, OBS_GLOBAL_TRANSITION_STATE,
|
||||
ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsReplicationStats, ObsStore, StorageAdminApi, obs_get_global_bucket_monitor,
|
||||
obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_load_data_usage_from_backend,
|
||||
|
||||
@@ -31,3 +31,12 @@ pub(crate) use rustfs_ecstore::api::global::{
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as ObsStore;
|
||||
pub(crate) use rustfs_storage_api::{BucketOperations, BucketOptions, StorageAdminApi};
|
||||
|
||||
pub(crate) mod metrics {
|
||||
pub(crate) use super::{
|
||||
BucketOperations, BucketOptions, OBS_GLOBAL_EXPIRY_STATE, OBS_GLOBAL_REPLICATION_STATS, OBS_GLOBAL_TRANSITION_STATE,
|
||||
ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsReplicationStats, ObsStore, StorageAdminApi,
|
||||
obs_get_global_bucket_monitor, obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free,
|
||||
obs_load_data_usage_from_backend, obs_resolve_object_store_handle,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
//! Swift account operations and validation
|
||||
|
||||
use super::storage_api::{BucketOperations, MakeBucketOptions};
|
||||
use super::storage_api::account::{BucketOperations, MakeBucketOptions};
|
||||
use super::{SwiftError, SwiftResult};
|
||||
use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata};
|
||||
use rustfs_credentials::Credentials;
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
//! This module implements Swift container CRUD operations and container-bucket translation.
|
||||
|
||||
use super::account::validate_account_access;
|
||||
use super::storage_api::{
|
||||
use super::storage_api::container::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListOperations as _, MakeBucketOptions,
|
||||
};
|
||||
use super::types::Container;
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
//! Segments are discovered at download time using lexicographic ordering
|
||||
//! based on a container metadata manifest pointer.
|
||||
|
||||
use super::storage_api::HTTPRangeSpec;
|
||||
use super::storage_api::large_object::HTTPRangeSpec;
|
||||
use super::{SwiftError, container, object};
|
||||
use axum::http::{HeaderMap, Response, StatusCode};
|
||||
use rustfs_credentials::Credentials;
|
||||
|
||||
@@ -61,7 +61,9 @@ pub mod versioning;
|
||||
pub use errors::{SwiftError, SwiftResult};
|
||||
pub use router::{SwiftRoute, SwiftRouter};
|
||||
// Note: Container, Object, and SwiftMetadata types used by Swift implementation
|
||||
pub use storage_api::{SwiftGetObjectReader, SwiftObjectInfo, SwiftObjectOptions, SwiftPutObjReader};
|
||||
pub(crate) use storage_api::{get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata};
|
||||
pub use storage_api::public_api::{SwiftGetObjectReader, SwiftObjectInfo, SwiftObjectOptions, SwiftPutObjReader};
|
||||
pub(crate) use storage_api::public_api::{
|
||||
get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata,
|
||||
};
|
||||
#[allow(unused_imports)]
|
||||
pub use types::{Container, Object, SwiftMetadata};
|
||||
|
||||
@@ -51,7 +51,7 @@
|
||||
|
||||
use super::account::validate_account_access;
|
||||
use super::container::ContainerMapper;
|
||||
use super::storage_api::{BucketOperations, BucketOptions, HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
|
||||
use super::storage_api::object::{BucketOperations, BucketOptions, HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
|
||||
use super::{SwiftError, SwiftResult, resolve_swift_object_store_handle};
|
||||
use axum::http::HeaderMap;
|
||||
use rustfs_credentials::Credentials;
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
//! Large files (>5GB) are split into segments, and a manifest defines
|
||||
//! how segments are assembled on download.
|
||||
|
||||
use super::storage_api::HTTPRangeSpec;
|
||||
use super::storage_api::large_object::HTTPRangeSpec;
|
||||
use super::{SwiftError, object};
|
||||
use axum::http::{HeaderMap, Response, StatusCode};
|
||||
use rustfs_credentials::Credentials;
|
||||
|
||||
@@ -26,6 +26,31 @@ pub(crate) use rustfs_storage_api::{
|
||||
ObjectOperations,
|
||||
};
|
||||
|
||||
pub(crate) mod account {
|
||||
pub(crate) use super::{BucketOperations, MakeBucketOptions};
|
||||
}
|
||||
|
||||
pub(crate) mod container {
|
||||
pub(crate) use super::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, ListOperations, MakeBucketOptions};
|
||||
}
|
||||
|
||||
pub(crate) mod large_object {
|
||||
pub(crate) use super::HTTPRangeSpec;
|
||||
}
|
||||
|
||||
pub(crate) mod object {
|
||||
pub(crate) use super::{BucketOperations, BucketOptions, HTTPRangeSpec, ObjectIO, ObjectOperations};
|
||||
}
|
||||
|
||||
pub(crate) mod public_api {
|
||||
pub use super::{SwiftGetObjectReader, SwiftObjectInfo, SwiftObjectOptions, SwiftPutObjReader};
|
||||
pub(crate) use super::{get_swift_bucket_metadata, resolve_swift_object_store_handle, set_swift_bucket_metadata};
|
||||
}
|
||||
|
||||
pub(crate) mod versioning {
|
||||
pub(crate) use super::{ListOperations, ObjectOperations};
|
||||
}
|
||||
|
||||
pub type SwiftGetObjectReader = <SwiftStore as ObjectIO>::GetObjectReader;
|
||||
pub type SwiftObjectInfo = <SwiftStore as ObjectOperations>::ObjectInfo;
|
||||
pub type SwiftObjectOptions = <SwiftStore as ObjectOperations>::ObjectOptions;
|
||||
|
||||
@@ -55,7 +55,7 @@ use super::account::validate_account_access;
|
||||
use super::container::ContainerMapper;
|
||||
use super::object::{ObjectKeyMapper, SwiftObjectOptions as ObjectOptions, head_object};
|
||||
use super::resolve_swift_object_store_handle;
|
||||
use super::storage_api::{ListOperations as _, ObjectOperations as _};
|
||||
use super::storage_api::versioning::{ListOperations as _, ObjectOperations as _};
|
||||
use super::{SwiftError, SwiftResult};
|
||||
use rustfs_credentials::Credentials;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
@@ -25,7 +25,7 @@ mod storage_api;
|
||||
mod test;
|
||||
|
||||
pub type QueryResult<T> = Result<T, QueryError>;
|
||||
pub(crate) use storage_api::{
|
||||
pub(crate) use storage_api::crate_boundary::{
|
||||
SELECT_DEFAULT_READ_BUFFER_SIZE, SelectGetObjectReader, SelectObjectInfo, SelectObjectOptions, SelectStorageError,
|
||||
SelectStore, resolve_select_object_store_handle, select_is_err_bucket_not_found, select_is_err_object_not_found,
|
||||
select_is_err_version_not_found,
|
||||
|
||||
@@ -48,7 +48,7 @@ use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio_util::io::ReaderStream;
|
||||
use transform_stream::AsyncTryStream;
|
||||
|
||||
use crate::storage_api::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
|
||||
use crate::storage_api::object_store::{HTTPRangeSpec, ObjectIO as _, ObjectOperations as _};
|
||||
|
||||
fn select_default_read_buffer_size_u64() -> u64 {
|
||||
u64::try_from(SELECT_DEFAULT_READ_BUFFER_SIZE).unwrap_or(u64::MAX)
|
||||
|
||||
@@ -25,6 +25,18 @@ pub(crate) use rustfs_ecstore::api::set_disk::DEFAULT_READ_BUFFER_SIZE as SELECT
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as SelectStore;
|
||||
pub(crate) use rustfs_storage_api::{HTTPRangeSpec, ObjectIO, ObjectOperations};
|
||||
|
||||
pub(crate) mod object_store {
|
||||
pub(crate) use super::{HTTPRangeSpec, ObjectIO, ObjectOperations};
|
||||
}
|
||||
|
||||
pub(crate) mod crate_boundary {
|
||||
pub(crate) use super::{
|
||||
SELECT_DEFAULT_READ_BUFFER_SIZE, SelectGetObjectReader, SelectObjectInfo, SelectObjectOptions, SelectStorageError,
|
||||
SelectStore, resolve_select_object_store_handle, select_is_err_bucket_not_found, select_is_err_object_not_found,
|
||||
select_is_err_version_not_found,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) type SelectGetObjectReader = <SelectStore as ObjectIO>::GetObjectReader;
|
||||
pub(crate) type SelectObjectInfo = <SelectStore as ObjectOperations>::ObjectInfo;
|
||||
pub(crate) type SelectObjectOptions = <SelectStore as ObjectOperations>::ObjectOptions;
|
||||
|
||||
@@ -24,7 +24,7 @@ use http::HeaderMap;
|
||||
use rustfs_config::server_config::{Config as ServerConfig, get_global_server_config as config_get_global_server_config};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use storage_api::{
|
||||
use storage_api::owner::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_RUSTFS_META_BUCKET, ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS,
|
||||
ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys, EcstoreDisk,
|
||||
EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions, EcstoreDiskLocation,
|
||||
@@ -40,7 +40,7 @@ use storage_api::{
|
||||
ecstore_resolve_object_store_handle, ecstore_save_config,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use storage_api::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk};
|
||||
use storage_api::owner::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
pub mod data_usage_define;
|
||||
|
||||
@@ -44,7 +44,7 @@ use tokio::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
|
||||
use crate::storage_api::{BucketOperations, BucketOptions, NamespaceLocking as _};
|
||||
use crate::storage_api::scan::{BucketOperations, BucketOptions, NamespaceLocking as _};
|
||||
use crate::{
|
||||
ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerLifecycleConfigExt as _, ScannerReplicationConfigExt as _,
|
||||
get_lifecycle_config, get_replication_config, read_config, replace_bucket_usage_memory_from_info, save_config,
|
||||
@@ -1148,9 +1148,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api::ObjectIO for MemoryConfigStore {
|
||||
impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
|
||||
type Error = EcstoreError;
|
||||
type RangeSpec = crate::storage_api::HTTPRangeSpec;
|
||||
type RangeSpec = crate::storage_api::scanner_io::HTTPRangeSpec;
|
||||
type HeaderMap = http::HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
@@ -1161,7 +1161,7 @@ mod tests {
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
_range: Option<crate::storage_api::HTTPRangeSpec>,
|
||||
_range: Option<crate::storage_api::scanner_io::HTTPRangeSpec>,
|
||||
_h: http::HeaderMap,
|
||||
_opts: &ObjectOptions,
|
||||
) -> EcstoreResult<GetObjectReader> {
|
||||
|
||||
@@ -42,7 +42,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use crate::ScannerObjectInfo as ObjectInfo;
|
||||
use crate::storage_api::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
|
||||
use crate::storage_api::scanner_io::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
|
||||
use crate::{
|
||||
BucketTargetSys, BucketVersioningSys, Disk, DiskError, ECStore, EcstoreError as Error, EcstoreResult as Result,
|
||||
ReplicationConfig, STORAGE_FORMAT_FILE, ScannerDiskExt as _, ScannerLifecycleConfigExt as _,
|
||||
|
||||
@@ -76,3 +76,34 @@ pub(crate) use rustfs_storage_api::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, HTTPRangeSpec, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
ObjectToDelete, StorageAdminApi,
|
||||
};
|
||||
|
||||
pub(crate) mod owner {
|
||||
pub(crate) use super::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_RUSTFS_META_BUCKET, ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS,
|
||||
ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE, EcstoreBucketTargetSys, EcstoreBucketVersioningSys,
|
||||
EcstoreDisk, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskInfo, EcstoreDiskInfoOptions,
|
||||
EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator, EcstoreEvent, EcstoreLcEventSrc,
|
||||
EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreObjectOpts, EcstoreReplicationConfig,
|
||||
EcstoreReplicationConfigurationExt, EcstoreReplicationHealQueueResult, EcstoreReplicationQueueAdmission,
|
||||
EcstoreResultType, EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig,
|
||||
EcstoreVersioningApi, HTTPRangeSpec, ObjectIO, ObjectOperations, ObjectToDelete, ecstore_apply_expiry_rule,
|
||||
ecstore_apply_transition_rule, ecstore_get_global_expiry_state, ecstore_get_global_tier_config_mgr,
|
||||
ecstore_get_lifecycle_config, ecstore_get_object_lock_config, ecstore_get_replication_config, ecstore_is_erasure,
|
||||
ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket, ecstore_list_path_raw, ecstore_path2_bucket_object,
|
||||
ecstore_path2_bucket_object_with_base_path, ecstore_queue_replication_heal_internal, ecstore_read_config,
|
||||
ecstore_replace_bucket_usage_memory_from_info, ecstore_resolve_object_store_handle, ecstore_save_config,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk};
|
||||
}
|
||||
|
||||
pub(crate) mod scan {
|
||||
pub(crate) use super::{BucketOperations, BucketOptions, NamespaceLocking};
|
||||
}
|
||||
|
||||
pub(crate) mod scanner_io {
|
||||
pub(crate) use super::{BucketInfo, BucketOperations, BucketOptions, DiskSetSelector, StorageAdminApi};
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::{HTTPRangeSpec, ObjectIO};
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ use uuid::Uuid;
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use storage_api::{
|
||||
use storage_api::lifecycle::{
|
||||
BUCKET_LIFECYCLE_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, CompletePart, DiskAPI as _, DiskOption,
|
||||
ECStore, Endpoint, EndpointServerPools, Endpoints, ListOperations as _, MakeBucketOptions, MultipartOperations as _,
|
||||
ObjectIO as _, ObjectOperations as _, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, ScannerWarmBackend,
|
||||
|
||||
@@ -35,3 +35,14 @@ pub(crate) use rustfs_storage_api::{
|
||||
BucketOperations, BucketOptions, CompletePart, ListOperations, MakeBucketOptions, MultipartOperations, ObjectIO,
|
||||
ObjectOperations,
|
||||
};
|
||||
|
||||
pub(crate) mod lifecycle {
|
||||
pub(crate) use super::{
|
||||
BUCKET_LIFECYCLE_CONFIG, BucketOperations, BucketOptions, BucketVersioningSys, CompletePart, DiskAPI, DiskOption,
|
||||
ECStore, Endpoint, EndpointServerPools, Endpoints, ListOperations, MakeBucketOptions, MultipartOperations, ObjectIO,
|
||||
ObjectOperations, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, ScannerWarmBackend, TierConfig, TierMinIO,
|
||||
TierType, TransitionOptions, WarmBackendGetOpts, build_transition_put_options, enqueue_transition_for_existing_objects,
|
||||
get_bucket_metadata, get_global_tier_config_mgr, init_background_expiry, init_bucket_metadata_sys, init_local_disks,
|
||||
new_disk, path2_bucket_object_with_base_path, update_bucket_metadata,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user