mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
refactor(runtime): route RustFS runtime consumers through storage owner (#3756)
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as E2eBucketTargetSys;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo as E2eVolumeInfo, WalkDirOptions as E2eWalkDirOptions};
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
TonicInterceptor as E2eTonicInterceptor, node_service_time_out_client_no_auth as e2e_node_service_time_out_client_no_auth,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
gen_tonic_signature_interceptor as gen_e2e_tonic_signature_interceptor,
|
||||
node_service_time_out_client as e2e_node_service_time_out_client,
|
||||
};
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod ecstore_test_compat;
|
||||
mod reliant;
|
||||
|
||||
// Common utilities for all E2E tests
|
||||
|
||||
@@ -15,8 +15,10 @@
|
||||
// Used by test_distributed_lock_4_nodes_grpc in lock.rs
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::ecstore_test_compat::{
|
||||
E2eTonicInterceptor as TonicInterceptor, e2e_node_service_time_out_client_no_auth as node_service_time_out_client_no_auth,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_ecstore::api::rpc::{TonicInterceptor, node_service_time_out_client_no_auth};
|
||||
use rustfs_lock::{
|
||||
LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
types::{LockMetadata, LockPriority},
|
||||
|
||||
@@ -14,10 +14,13 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::workspace_root;
|
||||
use crate::ecstore_test_compat::{
|
||||
E2eTonicInterceptor as TonicInterceptor, E2eVolumeInfo as VolumeInfo, E2eWalkDirOptions as WalkDirOptions,
|
||||
e2e_node_service_time_out_client as node_service_time_out_client,
|
||||
gen_e2e_tonic_signature_interceptor as gen_tonic_signature_interceptor,
|
||||
};
|
||||
use futures::future::join_all;
|
||||
use rmp_serde::{Deserializer, Serializer};
|
||||
use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
|
||||
use rustfs_ecstore::api::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
|
||||
use rustfs_protos::proto_gen::node_service::WalkDirRequest;
|
||||
use rustfs_protos::{
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use crate::common::{
|
||||
RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
|
||||
};
|
||||
use crate::ecstore_test_compat::E2eBucketTargetSys as BucketTargetSys;
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
@@ -22,7 +23,6 @@ use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
|
||||
use rustfs_madmin::{
|
||||
AddServiceAccountReq, ListServiceAccountsResp, PeerInfo, PeerSite, ReplicateAddStatus, ReplicateEditStatus,
|
||||
ReplicateRemoveStatus, SRRemoveReq, SRResyncOpStatus, SRStatusInfo, SiteReplicationInfo, SyncStatus,
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME;
|
||||
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
|
||||
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, DeleteOptions as EcstoreDeleteOptions,
|
||||
DiskAPI as EcstoreDiskAPI, DiskStore as EcstoreDiskStore, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
|
||||
pub(crate) use rustfs_ecstore::api::global::GLOBAL_LOCAL_DISK_MAP as ECSTORE_GLOBAL_LOCAL_DISK_MAP;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
@@ -22,19 +22,15 @@ pub mod storage;
|
||||
pub mod task;
|
||||
pub mod utils;
|
||||
|
||||
use rustfs_ecstore::api::data_usage::DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME;
|
||||
use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
|
||||
use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
|
||||
use rustfs_ecstore::api::disk::{
|
||||
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, DeleteOptions as EcstoreDeleteOptions,
|
||||
DiskAPI as EcstoreDiskAPI, DiskStore as EcstoreDiskStore, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET,
|
||||
use ecstore_compat::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_GLOBAL_LOCAL_DISK_MAP, ECSTORE_RUSTFS_META_BUCKET,
|
||||
EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore,
|
||||
EcstoreEndpoint, EcstoreErrorType, EcstoreStorageError, EcstoreStore,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
|
||||
use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
|
||||
use rustfs_ecstore::api::global::GLOBAL_LOCAL_DISK_MAP as ECSTORE_GLOBAL_LOCAL_DISK_MAP;
|
||||
use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
use ecstore_compat::{EcstoreDiskOption, ecstore_new_disk};
|
||||
|
||||
mod ecstore_compat;
|
||||
pub use erasure_healer::ErasureSetHealer;
|
||||
pub use manager::{HealManager, HealOperationsSnapshot, HealPriorityCounts, HealSourceCounts};
|
||||
pub use resume::{CheckpointManager, ResumeCheckpoint, ResumeManager, ResumeState, ResumeUtils};
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
#![allow(unused_imports)]
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys;
|
||||
pub(crate) use rustfs_ecstore::api::disk::DiskStore;
|
||||
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
|
||||
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
|
||||
@@ -14,9 +14,9 @@
|
||||
|
||||
//! test endpoint index settings
|
||||
|
||||
use rustfs_ecstore::api::disk::endpoint::Endpoint;
|
||||
use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
|
||||
mod ecstore_test_compat;
|
||||
|
||||
use ecstore_test_compat::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks};
|
||||
use std::net::SocketAddr;
|
||||
use tempfile::TempDir;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -12,7 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_ecstore::api::disk::{DiskStore, endpoint::Endpoint};
|
||||
mod ecstore_test_compat;
|
||||
|
||||
use ecstore_test_compat::{DiskStore, Endpoint};
|
||||
use rustfs_heal::heal::{
|
||||
event::{HealEvent, Severity},
|
||||
task::{HealPriority, HealType},
|
||||
|
||||
@@ -12,12 +12,13 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod ecstore_test_compat;
|
||||
|
||||
use ecstore_test_compat::{
|
||||
ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_ecstore::api::bucket::metadata_sys::init_bucket_metadata_sys;
|
||||
use rustfs_ecstore::api::disk::endpoint::Endpoint;
|
||||
use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
|
||||
use rustfs_heal::heal::{
|
||||
manager::{HealConfig, HealManager},
|
||||
storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI},
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::config::RUSTFS_CONFIG_PREFIX as ECSTORE_RUSTFS_CONFIG_PREFIX;
|
||||
pub(crate) use rustfs_ecstore::api::config::com::{
|
||||
delete_config as ecstore_delete_config, read_config_no_lock as ecstore_read_config_no_lock,
|
||||
read_config_with_metadata as ecstore_read_config_with_metadata, save_config as ecstore_save_config,
|
||||
save_config_with_opts as ecstore_save_config_with_opts,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::error::{
|
||||
Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
|
||||
classify_system_path_failure_reason as ecstore_classify_system_path_failure_reason,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::global::is_first_cluster_node_local as ecstore_is_first_cluster_node_local;
|
||||
pub(crate) use rustfs_ecstore::api::notification::{
|
||||
NotificationPeerErr as EcstoreNotificationPeerErr, get_global_notification_sys as ecstore_get_global_notification_sys,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
+7
-15
@@ -13,23 +13,14 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use ecstore_compat::{
|
||||
ECSTORE_RUSTFS_CONFIG_PREFIX, EcstoreErrorType, EcstoreNotificationPeerErr, EcstoreResultType, EcstoreStorageError,
|
||||
EcstoreStore, ecstore_classify_system_path_failure_reason, ecstore_delete_config, ecstore_get_global_notification_sys,
|
||||
ecstore_is_first_cluster_node_local, ecstore_read_config_no_lock, ecstore_read_config_with_metadata, ecstore_save_config,
|
||||
ecstore_save_config_with_opts,
|
||||
};
|
||||
use manager::IamCache;
|
||||
use oidc::OidcSys;
|
||||
use rustfs_ecstore::api::config::RUSTFS_CONFIG_PREFIX as ECSTORE_RUSTFS_CONFIG_PREFIX;
|
||||
use rustfs_ecstore::api::config::com::{
|
||||
delete_config as ecstore_delete_config, read_config_no_lock as ecstore_read_config_no_lock,
|
||||
read_config_with_metadata as ecstore_read_config_with_metadata, save_config as ecstore_save_config,
|
||||
save_config_with_opts as ecstore_save_config_with_opts,
|
||||
};
|
||||
use rustfs_ecstore::api::error::{
|
||||
Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
|
||||
classify_system_path_failure_reason as ecstore_classify_system_path_failure_reason,
|
||||
};
|
||||
use rustfs_ecstore::api::global::is_first_cluster_node_local as ecstore_is_first_cluster_node_local;
|
||||
use rustfs_ecstore::api::notification::{
|
||||
NotificationPeerErr as EcstoreNotificationPeerErr, get_global_notification_sys as ecstore_get_global_notification_sys,
|
||||
};
|
||||
use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use store::object::ObjectStore;
|
||||
use sys::IamSys;
|
||||
@@ -42,6 +33,7 @@ const EVENT_IAM_STATE: &str = "iam_state";
|
||||
const EVENT_OIDC_STATE: &str = "oidc_state";
|
||||
|
||||
pub mod cache;
|
||||
mod ecstore_compat;
|
||||
pub mod error;
|
||||
pub mod keyring;
|
||||
pub mod manager;
|
||||
|
||||
@@ -20,6 +20,13 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_ecstore::api::config::com::{
|
||||
read_config_without_migrate as read_notify_config_without_migrate_from_backend,
|
||||
save_server_config as save_notify_server_config_to_backend,
|
||||
};
|
||||
use rustfs_ecstore::api::global::resolve_object_store_handle as resolve_notify_object_store_handle_from_backend;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as NotifyStore;
|
||||
|
||||
mod bucket_config_manager;
|
||||
mod config_manager;
|
||||
mod error;
|
||||
@@ -39,8 +46,6 @@ mod runtime_view;
|
||||
mod services;
|
||||
mod status_view;
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as NotifyStore;
|
||||
|
||||
pub use bucket_config_manager::NotifyBucketConfigManager;
|
||||
pub use config_manager::{NotifyConfigManager, runtime_target_id_for_subsystem};
|
||||
pub use error::{LifecycleError, NotificationError};
|
||||
@@ -60,13 +65,13 @@ pub use services::NotifyServices;
|
||||
pub use status_view::NotifyStatusView;
|
||||
|
||||
pub(crate) fn resolve_notify_object_store_handle() -> Option<Arc<NotifyStore>> {
|
||||
rustfs_ecstore::api::global::resolve_object_store_handle()
|
||||
resolve_notify_object_store_handle_from_backend()
|
||||
}
|
||||
|
||||
pub(crate) async fn read_notify_server_config_without_migrate(
|
||||
store: Arc<NotifyStore>,
|
||||
) -> Result<rustfs_config::server_config::Config, String> {
|
||||
rustfs_ecstore::api::config::com::read_config_without_migrate(store)
|
||||
read_notify_config_without_migrate_from_backend(store)
|
||||
.await
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
@@ -75,7 +80,7 @@ pub(crate) async fn save_notify_server_config(
|
||||
store: Arc<NotifyStore>,
|
||||
config: &rustfs_config::server_config::Config,
|
||||
) -> Result<(), String> {
|
||||
rustfs_ecstore::api::config::com::save_server_config(store, config)
|
||||
save_notify_server_config_to_backend(store, config)
|
||||
.await
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
@@ -12,13 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod collectors;
|
||||
pub mod config;
|
||||
pub mod report;
|
||||
pub mod scheduler;
|
||||
pub mod schema;
|
||||
pub mod stats_collector;
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
GLOBAL_ExpiryState as OBS_GLOBAL_EXPIRY_STATE, GLOBAL_TransitionState as OBS_GLOBAL_TRANSITION_STATE,
|
||||
};
|
||||
@@ -35,6 +28,13 @@ pub(crate) use rustfs_ecstore::api::global::{
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as ObsStore;
|
||||
|
||||
pub mod collectors;
|
||||
pub mod config;
|
||||
pub mod report;
|
||||
pub mod scheduler;
|
||||
pub mod schema;
|
||||
pub mod stats_collector;
|
||||
|
||||
pub use collectors::*;
|
||||
pub use config::*;
|
||||
pub use report::{PrometheusMetric, report_metrics};
|
||||
|
||||
@@ -35,12 +35,12 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_ecstore::api::bucket::metadata::BucketMetadata as SwiftBucketMetadata;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata as SwiftBucketMetadata;
|
||||
use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
get as get_swift_bucket_metadata_from_backend, set_bucket_metadata as set_swift_bucket_metadata_in_backend,
|
||||
};
|
||||
use rustfs_ecstore::api::error::Result as SwiftStorageResult;
|
||||
use rustfs_ecstore::api::global::resolve_object_store_handle as resolve_swift_object_store_handle;
|
||||
pub(crate) use rustfs_ecstore::api::error::Result as SwiftStorageResult;
|
||||
pub(crate) use rustfs_ecstore::api::global::resolve_object_store_handle as resolve_swift_object_store_handle;
|
||||
use rustfs_ecstore::api::storage::ECStore as SwiftStore;
|
||||
|
||||
pub mod account;
|
||||
@@ -78,10 +78,10 @@ pub type SwiftObjectInfo = <SwiftStore as rustfs_storage_api::ObjectOperations>:
|
||||
pub type SwiftObjectOptions = <SwiftStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
|
||||
pub type SwiftPutObjReader = <SwiftStore as rustfs_storage_api::ObjectIO>::PutObjectReader;
|
||||
|
||||
async fn get_swift_bucket_metadata(bucket: &str) -> SwiftStorageResult<Arc<SwiftBucketMetadata>> {
|
||||
pub(crate) async fn get_swift_bucket_metadata(bucket: &str) -> SwiftStorageResult<Arc<SwiftBucketMetadata>> {
|
||||
get_swift_bucket_metadata_from_backend(bucket).await
|
||||
}
|
||||
|
||||
async fn set_swift_bucket_metadata(bucket: String, metadata: SwiftBucketMetadata) -> SwiftStorageResult<()> {
|
||||
pub(crate) async fn set_swift_bucket_metadata(bucket: String, metadata: SwiftBucketMetadata) -> SwiftStorageResult<()> {
|
||||
set_swift_bucket_metadata_in_backend(bucket, metadata).await
|
||||
}
|
||||
|
||||
@@ -13,12 +13,17 @@
|
||||
// limitations under the License.
|
||||
|
||||
use datafusion::{common::DataFusionError, sql::sqlparser::parser::ParserError};
|
||||
use snafu::{Backtrace, Location, Snafu};
|
||||
use std::fmt::Display;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::error::StorageError as SelectStorageError;
|
||||
use rustfs_ecstore::api::error::{
|
||||
is_err_bucket_not_found as select_is_err_bucket_not_found_from_backend,
|
||||
is_err_object_not_found as select_is_err_object_not_found_from_backend,
|
||||
is_err_version_not_found as select_is_err_version_not_found_from_backend,
|
||||
};
|
||||
use rustfs_ecstore::api::global::resolve_object_store_handle as resolve_select_object_store_handle_from_backend;
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::DEFAULT_READ_BUFFER_SIZE as SELECT_DEFAULT_READ_BUFFER_SIZE;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as SelectStore;
|
||||
use snafu::{Backtrace, Location, Snafu};
|
||||
use std::{fmt::Display, sync::Arc};
|
||||
|
||||
pub mod object_store;
|
||||
pub mod query;
|
||||
@@ -33,22 +38,20 @@ pub(crate) type SelectGetObjectReader = <SelectStore as rustfs_storage_api::Obje
|
||||
pub(crate) type SelectObjectInfo = <SelectStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
|
||||
pub(crate) type SelectObjectOptions = <SelectStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
|
||||
|
||||
pub(crate) const SELECT_DEFAULT_READ_BUFFER_SIZE: usize = rustfs_ecstore::api::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
|
||||
pub(crate) fn resolve_select_object_store_handle() -> Option<Arc<SelectStore>> {
|
||||
rustfs_ecstore::api::global::resolve_object_store_handle()
|
||||
resolve_select_object_store_handle_from_backend()
|
||||
}
|
||||
|
||||
pub(crate) fn select_is_err_bucket_not_found(err: &SelectStorageError) -> bool {
|
||||
rustfs_ecstore::api::error::is_err_bucket_not_found(err)
|
||||
select_is_err_bucket_not_found_from_backend(err)
|
||||
}
|
||||
|
||||
pub(crate) fn select_is_err_object_not_found(err: &SelectStorageError) -> bool {
|
||||
rustfs_ecstore::api::error::is_err_object_not_found(err)
|
||||
select_is_err_object_not_found_from_backend(err)
|
||||
}
|
||||
|
||||
pub(crate) fn select_is_err_version_not_found(err: &SelectStorageError) -> bool {
|
||||
rustfs_ecstore::api::error::is_err_version_not_found(err)
|
||||
select_is_err_version_not_found_from_backend(err)
|
||||
}
|
||||
|
||||
#[derive(Debug, Snafu)]
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
GLOBAL_ExpiryState as ECSTORE_GLOBAL_EXPIRY_STATE, apply_expiry_rule as ecstore_apply_expiry_rule,
|
||||
apply_transition_rule as ecstore_apply_transition_rule,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator as EcstoreEvaluator;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{
|
||||
Event as EcstoreEvent, Lifecycle as EcstoreLifecycle, ObjectOpts as EcstoreObjectOpts,
|
||||
TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
|
||||
get_replication_config as ecstore_get_replication_config,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::replication::{
|
||||
ReplicationConfig as EcstoreReplicationConfig, ReplicationConfigurationExt as EcstoreReplicationConfigurationExt,
|
||||
ReplicationHealQueueResult as EcstoreReplicationHealQueueResult,
|
||||
ReplicationQueueAdmission as EcstoreReplicationQueueAdmission,
|
||||
queue_replication_heal_internal as ecstore_queue_replication_heal_internal,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::versioning::VersioningApi as EcstoreVersioningApi;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys as EcstoreBucketVersioningSys;
|
||||
pub(crate) use rustfs_ecstore::api::cache::{
|
||||
ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::capacity::{
|
||||
is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object,
|
||||
path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::config::com::{read_config as ecstore_read_config, save_config as ecstore_save_config};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::config::init as ecstore_config_init;
|
||||
pub(crate) use rustfs_ecstore::api::config::storageclass::{
|
||||
RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::data_usage::replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
|
||||
pub(crate) use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, Disk as EcstoreDisk, DiskAPI as EcstoreDiskAPI,
|
||||
DiskInfo as EcstoreDiskInfo, DiskInfoOptions as EcstoreDiskInfoOptions, DiskLocation as EcstoreDiskLocation,
|
||||
RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE,
|
||||
ScanGuard as EcstoreScanGuard,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
DiskOption as EcstoreDiskOption, DiskStore as EcstoreDiskStore, new_disk as ecstore_new_disk,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::error::{
|
||||
Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::global::{
|
||||
GLOBAL_TierConfigMgr as ECSTORE_GLOBAL_TIER_CONFIG_MGR, is_erasure as ecstore_is_erasure,
|
||||
is_erasure_sd as ecstore_is_erasure_sd, resolve_object_store_handle as ecstore_resolve_object_store_handle,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
pub(crate) use rustfs_ecstore::api::tier::tier_config::TierConfig as EcstoreTierConfig;
|
||||
+18
-52
@@ -20,59 +20,25 @@
|
||||
rust_2018_idioms
|
||||
)]
|
||||
|
||||
mod ecstore_compat;
|
||||
|
||||
use ecstore_compat::{
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_GLOBAL_EXPIRY_STATE, ECSTORE_GLOBAL_TIER_CONFIG_MGR, ECSTORE_RUSTFS_META_BUCKET,
|
||||
ECSTORE_STORAGE_FORMAT_FILE, ECSTORE_STORAGECLASS_RRS, ECSTORE_STORAGECLASS_STANDARD, ECSTORE_TRANSITION_COMPLETE,
|
||||
EcstoreBucketTargetSys, EcstoreBucketVersioningSys, EcstoreDisk, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreDiskError,
|
||||
EcstoreDiskInfo, EcstoreDiskInfoOptions, EcstoreDiskLocation, EcstoreDiskResult, EcstoreErrorType, EcstoreEvaluator,
|
||||
EcstoreEvent, EcstoreLcEventSrc, EcstoreLifecycle, EcstoreListPathRawOptions, EcstoreObjectOpts, EcstoreReplicationConfig,
|
||||
EcstoreReplicationConfigurationExt, EcstoreReplicationHealQueueResult, EcstoreReplicationQueueAdmission, EcstoreResultType,
|
||||
EcstoreScanGuard, EcstoreSetDisks, EcstoreStorageError, EcstoreStore, EcstoreTierConfig, EcstoreVersioningApi,
|
||||
ecstore_apply_expiry_rule, ecstore_apply_transition_rule, ecstore_get_lifecycle_config, ecstore_get_object_lock_config,
|
||||
ecstore_get_replication_config, ecstore_is_erasure, ecstore_is_erasure_sd, ecstore_is_reserved_or_invalid_bucket,
|
||||
ecstore_list_path_raw, ecstore_path2_bucket_object, ecstore_path2_bucket_object_with_base_path,
|
||||
ecstore_queue_replication_heal_internal, ecstore_read_config, ecstore_replace_bucket_usage_memory_from_info,
|
||||
ecstore_resolve_object_store_handle, ecstore_save_config,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use ecstore_compat::{EcstoreDiskOption, EcstoreDiskStore, EcstoreEndpoint, ecstore_config_init, ecstore_new_disk};
|
||||
use http::HeaderMap;
|
||||
use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys as EcstoreBucketTargetSys;
|
||||
use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc as EcstoreLcEventSrc;
|
||||
use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
GLOBAL_ExpiryState as ECSTORE_GLOBAL_EXPIRY_STATE, apply_expiry_rule as ecstore_apply_expiry_rule,
|
||||
apply_transition_rule as ecstore_apply_transition_rule,
|
||||
};
|
||||
use rustfs_ecstore::api::bucket::lifecycle::evaluator::Evaluator as EcstoreEvaluator;
|
||||
use rustfs_ecstore::api::bucket::lifecycle::lifecycle::{
|
||||
Event as EcstoreEvent, Lifecycle as EcstoreLifecycle, ObjectOpts as EcstoreObjectOpts,
|
||||
TRANSITION_COMPLETE as ECSTORE_TRANSITION_COMPLETE,
|
||||
};
|
||||
use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
get_lifecycle_config as ecstore_get_lifecycle_config, get_object_lock_config as ecstore_get_object_lock_config,
|
||||
get_replication_config as ecstore_get_replication_config,
|
||||
};
|
||||
use rustfs_ecstore::api::bucket::replication::{
|
||||
ReplicationConfig as EcstoreReplicationConfig, ReplicationConfigurationExt as EcstoreReplicationConfigurationExt,
|
||||
ReplicationHealQueueResult as EcstoreReplicationHealQueueResult,
|
||||
ReplicationQueueAdmission as EcstoreReplicationQueueAdmission,
|
||||
queue_replication_heal_internal as ecstore_queue_replication_heal_internal,
|
||||
};
|
||||
use rustfs_ecstore::api::bucket::versioning::VersioningApi as EcstoreVersioningApi;
|
||||
use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys as EcstoreBucketVersioningSys;
|
||||
use rustfs_ecstore::api::cache::{ListPathRawOptions as EcstoreListPathRawOptions, list_path_raw as ecstore_list_path_raw};
|
||||
use rustfs_ecstore::api::capacity::{
|
||||
is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket, path2_bucket_object as ecstore_path2_bucket_object,
|
||||
path2_bucket_object_with_base_path as ecstore_path2_bucket_object_with_base_path,
|
||||
};
|
||||
use rustfs_ecstore::api::config::com::{read_config as ecstore_read_config, save_config as ecstore_save_config};
|
||||
#[cfg(test)]
|
||||
use rustfs_ecstore::api::config::init as ecstore_config_init;
|
||||
use rustfs_ecstore::api::config::storageclass::{RRS as ECSTORE_STORAGECLASS_RRS, STANDARD as ECSTORE_STORAGECLASS_STANDARD};
|
||||
use rustfs_ecstore::api::data_usage::replace_bucket_usage_memory_from_info as ecstore_replace_bucket_usage_memory_from_info;
|
||||
#[cfg(test)]
|
||||
use rustfs_ecstore::api::disk::endpoint::Endpoint as EcstoreEndpoint;
|
||||
use rustfs_ecstore::api::disk::error::{DiskError as EcstoreDiskError, Result as EcstoreDiskResult};
|
||||
use rustfs_ecstore::api::disk::{
|
||||
BUCKET_META_PREFIX as ECSTORE_BUCKET_META_PREFIX, Bytes as EcstoreDiskBytes, Disk as EcstoreDisk, DiskAPI as EcstoreDiskAPI,
|
||||
DiskInfo as EcstoreDiskInfo, DiskInfoOptions as EcstoreDiskInfoOptions, DiskLocation as EcstoreDiskLocation,
|
||||
RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET, STORAGE_FORMAT_FILE as ECSTORE_STORAGE_FORMAT_FILE,
|
||||
ScanGuard as EcstoreScanGuard,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, DiskStore as EcstoreDiskStore, new_disk as ecstore_new_disk};
|
||||
use rustfs_ecstore::api::error::{Error as EcstoreErrorType, Result as EcstoreResultType, StorageError as EcstoreStorageError};
|
||||
use rustfs_ecstore::api::global::{
|
||||
GLOBAL_TierConfigMgr as ECSTORE_GLOBAL_TIER_CONFIG_MGR, is_erasure as ecstore_is_erasure,
|
||||
is_erasure_sd as ecstore_is_erasure_sd, resolve_object_store_handle as ecstore_resolve_object_store_handle,
|
||||
};
|
||||
use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
|
||||
use rustfs_ecstore::api::storage::ECStore as EcstoreStore;
|
||||
use rustfs_ecstore::api::tier::tier_config::TierConfig as EcstoreTierConfig;
|
||||
use rustfs_storage_api::{HTTPRangeSpec, ObjectIO, ObjectToDelete};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
enqueue_transition_for_existing_objects, init_background_expiry,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::lifecycle::lifecycle::TransitionOptions;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::{
|
||||
get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::versioning_sys::BucketVersioningSys;
|
||||
pub(crate) use rustfs_ecstore::api::capacity::path2_bucket_object_with_base_path;
|
||||
pub(crate) use rustfs_ecstore::api::client::transition_api::{ReadCloser, ReaderImpl};
|
||||
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::global::GLOBAL_TierConfigMgr;
|
||||
pub(crate) use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
|
||||
pub(crate) use rustfs_ecstore::api::tier::tier_config::{TierConfig, TierMinIO, TierType};
|
||||
pub(crate) use rustfs_ecstore::api::tier::warm_backend::{
|
||||
WarmBackend as ScannerWarmBackend, WarmBackendGetOpts, build_transition_put_options,
|
||||
};
|
||||
@@ -12,28 +12,17 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod ecstore_test_compat;
|
||||
|
||||
use ecstore_test_compat::{
|
||||
BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DiskAPI as _, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
|
||||
GLOBAL_TierConfigMgr, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, ScannerWarmBackend, TierConfig, TierMinIO,
|
||||
TierType, TransitionOptions, WarmBackendGetOpts, build_transition_put_options, enqueue_transition_for_existing_objects,
|
||||
get_bucket_metadata, init_background_expiry, init_bucket_metadata_sys, init_local_disks, new_disk,
|
||||
path2_bucket_object_with_base_path, update_bucket_metadata,
|
||||
};
|
||||
use futures::FutureExt;
|
||||
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
|
||||
use rustfs_ecstore::api::bucket::{
|
||||
lifecycle::{
|
||||
bucket_lifecycle_ops::{enqueue_transition_for_existing_objects, init_background_expiry},
|
||||
lifecycle::TransitionOptions,
|
||||
},
|
||||
metadata::BUCKET_LIFECYCLE_CONFIG,
|
||||
metadata_sys::{get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata},
|
||||
versioning_sys::BucketVersioningSys,
|
||||
};
|
||||
use rustfs_ecstore::api::capacity::path2_bucket_object_with_base_path;
|
||||
use rustfs_ecstore::api::client::transition_api::{ReadCloser, ReaderImpl};
|
||||
use rustfs_ecstore::api::disk::endpoint::Endpoint;
|
||||
use rustfs_ecstore::api::disk::{DiskAPI as _, DiskOption, STORAGE_FORMAT_FILE, new_disk};
|
||||
use rustfs_ecstore::api::global::GLOBAL_TierConfigMgr;
|
||||
use rustfs_ecstore::api::layout::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use rustfs_ecstore::api::storage::{ECStore, init_local_disks};
|
||||
use rustfs_ecstore::api::tier::{
|
||||
tier_config::{TierConfig, TierMinIO, TierType},
|
||||
warm_backend::{WarmBackend as ScannerWarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
};
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_scanner::scanner_folder::ScannerItem;
|
||||
use rustfs_scanner::scanner_io::ScannerIODisk;
|
||||
|
||||
@@ -5,16 +5,20 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
## Current Context
|
||||
|
||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||
- Branch: `overtrue/arch-test-fuzz-owner-symbols`
|
||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134`.
|
||||
- Based on: API-134 slice.
|
||||
- Branch: `overtrue/arch-runtime-crate-ecstore-thin-bridges`
|
||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150`.
|
||||
- Based on: API-150 stacked slice.
|
||||
- PR type for this branch: `pure-move`
|
||||
- Runtime behavior changes: none.
|
||||
- Rust code changes: replace e2e, heal/scanner integration-test, and fuzz-target
|
||||
ECStore owner module aliases with explicit symbols.
|
||||
- CI/script changes: treat the completed test/fuzz files as explicit-symbol
|
||||
boundaries and reject restored `ecstore_*` module aliases there.
|
||||
- Docs changes: record the API-135 test/fuzz owner facade symbol cleanup.
|
||||
- Rust code changes: remove the thin notify/obs/swift/s3select ECStore
|
||||
compatibility bridge files and keep their ECStore source imports at each
|
||||
owner root.
|
||||
- CI/script changes: lock completed owner and test/fuzz boundaries against
|
||||
bare/glob imports, scattered raw ECStore facade subpaths, and startup
|
||||
runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade
|
||||
regressions, plus external runtime, test, fuzz, and storage-owner module
|
||||
ECStore compatibility bypasses, plus runtime crate thin bridge regressions.
|
||||
- Docs changes: record the API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151 owner facade cleanup.
|
||||
|
||||
## Phase 0 Tasks
|
||||
|
||||
@@ -3938,6 +3942,255 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
diff hygiene, Rust risk scan, branch freshness check, pre-commit quality
|
||||
gate, and three-expert review.
|
||||
|
||||
- [x] `API-136` Replace RustFS runtime owner module aliases with symbols.
|
||||
- Do: replace RustFS app/admin/storage owner-root `ecstore_*` facade aliases
|
||||
with owner-local curated symbol modules that expose only the ECStore
|
||||
submodules, functions, types, and constants consumed by those runtime
|
||||
boundaries.
|
||||
- Acceptance: `rustfs/src/app/mod.rs`, `rustfs/src/admin/mod.rs`, and
|
||||
`rustfs/src/storage/mod.rs` no longer import broad ECStore facade modules
|
||||
as `ecstore_*`; migration guards reject reintroducing those broad aliases.
|
||||
- Must preserve: app object/lifecycle/replication helpers, admin config,
|
||||
metrics, tiering, rebalance helpers, storage S3/RPC metadata helpers, and
|
||||
startup/server consumers of the storage owner boundary.
|
||||
- Verification: focused RustFS compile, runtime owner alias residual scan,
|
||||
migration/layer guards, formatting, diff hygiene, Rust risk scan, branch
|
||||
freshness check, pre-commit quality gate, and three-expert review.
|
||||
|
||||
- [x] `API-137` Guard completed owner facade import shapes.
|
||||
- Do: extend migration rules so completed owner and test/fuzz boundaries
|
||||
cannot reintroduce bare `rustfs_ecstore::api::<module>` imports or glob
|
||||
facade imports.
|
||||
- Acceptance: completed owner roots and completed test/fuzz boundaries keep
|
||||
explicit symbol imports, type aliases, constants, or wrappers; migration
|
||||
guards reject bare module and glob facade imports.
|
||||
- Must preserve: all API-136 RustFS owner symbol boundaries, API-135 test/fuzz
|
||||
direct symbol imports, and external owner root symbol imports.
|
||||
- Verification: architecture migration guard, shell syntax check, formatting,
|
||||
diff hygiene, branch freshness check, and three-expert review.
|
||||
|
||||
- [x] `API-138` Centralize completed owner raw facade subpaths.
|
||||
- Do: move completed notify and S3 Select owner wrapper raw ECStore facade
|
||||
calls into explicit import declarations, then guard completed owner and
|
||||
test/fuzz boundaries against raw facade subpaths outside import
|
||||
declarations.
|
||||
- Acceptance: completed owner and test/fuzz files keep raw ECStore facade
|
||||
subpaths centralized at import declarations; wrapper bodies use local
|
||||
aliases, constants, or functions.
|
||||
- Must preserve: notify config read/save wrappers, S3 Select object-store
|
||||
handle/error helpers, default read-buffer constant, and all existing public
|
||||
crate APIs.
|
||||
- Verification: focused notify/S3 Select compile, architecture migration
|
||||
guard, shell syntax check, formatting, diff hygiene, Rust risk scan, branch
|
||||
freshness check, and three-expert review.
|
||||
|
||||
- [x] `API-139` Route startup runtime consumers through storage owner symbols.
|
||||
- Do: expose storage-owner aliases and wrappers for startup layout, global
|
||||
endpoint/region state, local disk initialization, config initialization,
|
||||
background replication, and notification setup, then migrate startup
|
||||
runtime files away from `ecstore_*` owner-module consumers.
|
||||
- Acceptance: `startup_notification`, `startup_fs_guard`, `startup_services`,
|
||||
`startup_server`, and `startup_storage` use storage-owner symbols and
|
||||
wrappers instead of `crate::storage::ecstore_*` modules; migration guards
|
||||
reject restoring those module consumers.
|
||||
- Must preserve: endpoint parsing, unsupported-filesystem policy checks,
|
||||
global endpoint/erasure state setup, local disk and lock-client
|
||||
initialization, config migration/retry behavior, readiness marking,
|
||||
background replication start, region/port registration, and notification
|
||||
initialization.
|
||||
- Verification: focused RustFS test-target compile, migration guard, shell
|
||||
syntax check, formatting, diff hygiene, Rust risk scan, branch freshness
|
||||
check, and three-expert review.
|
||||
|
||||
- [x] `API-140` Route server/capacity/workload consumers through storage owner symbols.
|
||||
- Do: expose storage-owner symbols for local disk enumeration, disk endpoint
|
||||
labels, RPC signature prefix/verification, bucket metadata runtime state,
|
||||
replication pool access, and replication queue counts, then migrate
|
||||
server, capacity, and workload-admission consumers away from `ecstore_*`
|
||||
owner modules.
|
||||
- Acceptance: `server/http.rs`, `capacity/service.rs`, and
|
||||
`workload_admission.rs` use storage-owner symbols and wrappers instead of
|
||||
`crate::storage::ecstore_*` modules; migration guards reject restoring
|
||||
those module consumers.
|
||||
- Must preserve: internode RPC signature verification, active HTTP request
|
||||
metrics, capacity manager disk discovery/labels, metadata workload state,
|
||||
replication active/queue counts, and all storage-owner backend calls.
|
||||
- Verification: focused RustFS test-target compile, migration guard, shell
|
||||
syntax check, formatting, diff hygiene, Rust risk scan, branch freshness
|
||||
check, and three-expert review.
|
||||
|
||||
- [x] `API-141` Route root/server runtime consumers through storage owner symbols.
|
||||
- Do: expose storage-owner aliases and wrappers for notification config,
|
||||
topology capability mapping, readiness globals, event dispatch hook
|
||||
installation, module-switch config persistence, endpoint test builders,
|
||||
and quota/error types, then migrate remaining root/server runtime
|
||||
consumers away from `ecstore_*` owner modules.
|
||||
- Acceptance: `init.rs`, `runtime_capabilities.rs`, `server/readiness.rs`,
|
||||
`server/event.rs`, `server/module_switch.rs`, and `error.rs` use
|
||||
storage-owner symbols and wrappers instead of
|
||||
`crate::storage::ecstore_*` modules; migration guards reject restoring
|
||||
those module consumers.
|
||||
- Must preserve: bucket notification preload behavior, topology capability
|
||||
labels, runtime readiness lock-quorum checks, live event dispatch,
|
||||
module-switch persistence semantics, S3 error conversion, and test
|
||||
endpoint construction.
|
||||
- Verification: focused RustFS test-target compile, migration guard, shell
|
||||
syntax check, formatting, diff hygiene, Rust risk scan, branch freshness
|
||||
check, and three-expert review.
|
||||
|
||||
- [x] `API-142` Route table/S3/startup consumers through storage owner symbols.
|
||||
- Do: expose storage-owner constants, aliases, and wrappers for table catalog
|
||||
metadata roots, catalog path hashing, metadata lookup, lock timeout,
|
||||
shutdown, bucket metadata migration/init, S3 etag conversion, and config
|
||||
test disk layout parsing, then migrate the remaining table/S3/startup
|
||||
consumers away from `ecstore_*` owner modules.
|
||||
- Acceptance: `startup_bucket_metadata.rs`, `startup_shutdown.rs`,
|
||||
`table_catalog.rs`, `storage/s3_api/bucket.rs`,
|
||||
`storage/s3_api/multipart.rs`, and `config/config_test.rs` use
|
||||
storage-owner symbols and wrappers instead of
|
||||
`crate::storage::ecstore_*` modules; migration guards reject restoring
|
||||
those module consumers.
|
||||
- Must preserve: startup bucket metadata and IAM migration order,
|
||||
replication resync initialization, background-service shutdown, S3 ETag
|
||||
rendering, table catalog reserved paths and metadata hash layout,
|
||||
table-bucket mutation guard behavior, catalog lock acquisition timeout,
|
||||
and config test disk-layout parsing.
|
||||
- Verification: focused RustFS test-target compile, migration guard, shell
|
||||
syntax check, formatting, diff hygiene, Rust risk scan, branch freshness
|
||||
check, and three-expert review.
|
||||
|
||||
- [x] `API-143` Route app shared runtime facade through storage owner symbols.
|
||||
- Do: expose storage-owner aliases and wrappers for app-shared ECStore,
|
||||
endpoint layout, rio readers, notification access, global object-store
|
||||
resolver, shared error helpers, storage-class validation, and test local
|
||||
disk initialization, then migrate the duplicate app facade entries to
|
||||
delegate to storage-owner symbols.
|
||||
- Acceptance: `rustfs/src/app/mod.rs` delegates shared IO/error/global/
|
||||
notification/storage wrappers to `crate::storage` owner symbols instead
|
||||
of duplicate `ecstore_*` calls; migration guards reject restoring those
|
||||
duplicate calls.
|
||||
- Must preserve: app context resolution, object-store resolver fallback,
|
||||
notification system access, rio reader boxing/wrapping, lock timeout,
|
||||
storage-class validation, S3 ETag rendering, and app test disk setup.
|
||||
- Verification: focused RustFS test-target compile, migration guard, shell
|
||||
syntax check, formatting, diff hygiene, Rust risk scan, branch freshness
|
||||
check, and three-expert review.
|
||||
|
||||
- [x] `API-144` Route app bucket facade source imports through storage owner re-exports.
|
||||
- Do: expose bucket target/lifecycle/target, client transition API, and
|
||||
storageclass through storage owner re-exports, then source the app bucket,
|
||||
client, and config facade entries through `crate::storage`.
|
||||
- Acceptance: `rustfs/src/app/mod.rs` no longer imports direct
|
||||
`rustfs_ecstore::api::{bucket,client,config}::` source paths; migration
|
||||
guards reject restoring those direct source paths.
|
||||
- Must preserve: bucket target, lifecycle, metadata, object lock,
|
||||
policy/quota/replication/tagging/target/versioning, transition reader, and
|
||||
storageclass compatibility paths.
|
||||
- Verification: focused RustFS test-target compile, migration guard, shell
|
||||
syntax check, formatting, diff hygiene, Rust risk scan, branch freshness
|
||||
check, pre-commit, and three-expert review.
|
||||
|
||||
- [x] `API-145` Route remaining app facade ECStore source imports through storage owner re-exports.
|
||||
- Do: expose app-needed admin, capacity, compression, data-usage, global, and
|
||||
tier modules through storage owner re-exports, then source the remaining
|
||||
app facade entries through `crate::storage`.
|
||||
- Acceptance: `rustfs/src/app/mod.rs` contains no direct
|
||||
`rustfs_ecstore::api::` source imports; migration guards reject restoring
|
||||
any direct ECStore API source path in the app facade.
|
||||
- Must preserve: server info, pool capacity summaries, compression checks,
|
||||
bucket usage memory accounting, global tier manager access, and tier
|
||||
config/warm backend compatibility paths.
|
||||
- Verification: focused RustFS test-target compile, migration guard, shell
|
||||
syntax check, formatting, diff hygiene, Rust risk scan, branch freshness
|
||||
check, pre-commit, and three-expert review.
|
||||
|
||||
- [x] `API-146` Route admin facade ECStore source imports through storage owner re-exports.
|
||||
- Do: expose admin-needed bucket, capacity, client, config, data-usage, disk,
|
||||
error, global, layout, metrics, notification, rebalance, RPC, storage, and
|
||||
tier symbols through storage owner re-exports, then source the admin facade
|
||||
through `crate::storage`.
|
||||
- Acceptance: `rustfs/src/admin/mod.rs` contains no direct
|
||||
`rustfs_ecstore::api::` source imports; migration guards reject restoring
|
||||
any direct ECStore API source path in the admin facade.
|
||||
- Must preserve: admin handler utilities, bucket controls, storage class
|
||||
updates, data usage reads, cluster/global metadata, metrics/notification
|
||||
views, rebalance status, peer RPC, ECStore handle, and tier admin paths.
|
||||
- Verification: focused RustFS test-target compile, migration guard, shell
|
||||
syntax check, formatting, diff hygiene, Rust risk scan, branch freshness
|
||||
check, pre-commit, and three-expert review.
|
||||
|
||||
- [x] `API-147` Route external runtime crate ECStore source imports through local compatibility boundaries.
|
||||
- Do: move direct ECStore facade source imports in notify, observability
|
||||
metrics, S3 Select, Swift, IAM, heal, and scanner runtime entry modules
|
||||
into crate-local `ecstore_compat` modules while preserving existing
|
||||
wrappers and aliases at each crate boundary.
|
||||
- Acceptance: target runtime crate source directories contain no direct
|
||||
`rustfs_ecstore::api::` source paths outside `ecstore_compat.rs`; migration
|
||||
guards reject restoring those bypasses.
|
||||
- Must preserve: notify config persistence and object-store resolution,
|
||||
observability storage/ILM/replication metrics, S3 Select storage error
|
||||
mapping and object reads, Swift metadata/object-store access, IAM config
|
||||
and notification behavior, heal disk wrappers, and scanner lifecycle/disk
|
||||
runtime wrappers.
|
||||
- Verification: focused external crate compile, migration guard, shell syntax
|
||||
check, formatting, diff hygiene, Rust risk scan, branch freshness check,
|
||||
pre-commit, and three-expert review.
|
||||
|
||||
- [x] `API-148` Route external test ECStore source imports through local compatibility boundaries.
|
||||
- Do: move direct ECStore facade source imports in heal integration tests,
|
||||
scanner lifecycle tests, and e2e reliant/replication helpers into local
|
||||
`ecstore_test_compat` modules while preserving existing test aliases and
|
||||
helper call paths.
|
||||
- Acceptance: target external test/e2e paths contain no direct
|
||||
`rustfs_ecstore::api::` source paths outside `ecstore_test_compat.rs`;
|
||||
migration guards reject restoring those bypasses.
|
||||
- Must preserve: heal endpoint setup and resume disk types, scanner
|
||||
lifecycle transition setup, e2e node RPC client helpers, and replication
|
||||
bucket target cleanup behavior.
|
||||
- Verification: focused test-target compile, migration guard, shell syntax
|
||||
check, formatting, diff hygiene, Rust risk scan, branch freshness check,
|
||||
pre-commit, and three-expert review.
|
||||
|
||||
- [x] `API-149` Route fuzz ECStore source imports through a local compatibility boundary.
|
||||
- Do: move direct ECStore facade source imports in bucket validation and path
|
||||
containment fuzz targets into `ecstore_fuzz_compat` wrapper functions.
|
||||
- Acceptance: fuzz targets contain no direct `rustfs_ecstore::api::` source
|
||||
paths outside `ecstore_fuzz_compat.rs`; migration guards reject restoring
|
||||
those bypasses.
|
||||
- Must preserve: bucket/object validation fuzz semantics, meta bucket
|
||||
compatibility checks, object prefix/path component validation, and root
|
||||
containment assertions.
|
||||
- Verification: focused fuzz compile, migration guard, shell syntax check,
|
||||
formatting, diff hygiene, Rust risk scan, branch freshness check,
|
||||
pre-commit, and three-expert review.
|
||||
|
||||
- [x] `API-150` Move storage owner ECStore source imports into a compatibility module.
|
||||
- Do: move the storage owner `ecstore_*` facade source modules out of
|
||||
`rustfs/src/storage/mod.rs` and into `rustfs/src/storage/ecstore_compat.rs`.
|
||||
- Acceptance: `rustfs/src/storage/mod.rs` contains no direct
|
||||
`rustfs_ecstore::api::` source paths, while existing `crate::storage::*`
|
||||
aliases and helper functions keep their public shape.
|
||||
- Must preserve: storage owner type aliases, constants, wrapper functions,
|
||||
disk RPC extension traits, bucket metadata helpers, runtime globals, and
|
||||
startup storage wiring.
|
||||
- Verification: RustFS compile coverage, migration guard, shell syntax check,
|
||||
formatting, diff hygiene, Rust risk scan, branch freshness check,
|
||||
pre-commit, and three-expert review.
|
||||
|
||||
- [x] `API-151` Collapse thin runtime crate ECStore compatibility bridges.
|
||||
- Do: remove the thin `ecstore_compat.rs` files from notify, obs metrics,
|
||||
Swift, and S3 Select, moving their aliases and wrappers to the owner root
|
||||
modules.
|
||||
- Acceptance: those crates no longer declare local `ecstore_compat` modules,
|
||||
while their public/internal owner-root aliases and wrapper functions keep
|
||||
the same call paths for downstream modules.
|
||||
- Must preserve: notify server-config IO, metrics data/quota/replication
|
||||
reads, S3 Select object-reader/error mapping, and Swift bucket metadata and
|
||||
object reader aliases.
|
||||
- Verification: focused runtime crate compile coverage, migration guard,
|
||||
shell syntax check, formatting, diff hygiene, Rust risk scan, branch
|
||||
freshness check, pre-commit, and three-expert review.
|
||||
|
||||
## Next PRs
|
||||
|
||||
1. `pure-move`: continue pruning remaining facade compatibility and owner boundaries.
|
||||
@@ -3946,14 +4199,213 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
|
||||
| Expert | Status | Notes |
|
||||
|---|---|---|
|
||||
| Quality/architecture | pass | API-135 replaces e2e, heal/scanner integration-test, and fuzz-target ECStore owner module aliases with explicit imported symbols. |
|
||||
| Migration preservation | pass | RPC client construction, replication target access, heal ECStore setup, scanner lifecycle/tier/transition paths, and fuzz validation call targets keep the same ECStore facade symbols. |
|
||||
| Testing/verification | pass | Focused e2e/heal/scanner compile, fuzz manifest compile, completed test/fuzz alias residual scan, migration/layer guards, formatting, diff hygiene, full pre-commit, and diff-only Rust risk scan passed. |
|
||||
| Quality/architecture | pass | API-151 removes thin runtime crate ECStore bridge files and keeps the remaining source imports at owner roots. |
|
||||
| Migration preservation | pass | Notify, obs metrics, Swift, and S3 Select keep the same owner-root aliases and wrappers for existing consumers. |
|
||||
| Testing/verification | pass | Focused runtime crate compile, formatting, migration guard, shell syntax, diff hygiene, Rust risk scan, and pre-commit passed for API-151. |
|
||||
|
||||
## Verification Notes
|
||||
|
||||
Passed before push:
|
||||
|
||||
- Issue #660 API-151 current slice:
|
||||
- `cargo check -p rustfs-notify -p rustfs-obs -p rustfs-s3select-api -p rustfs-protocols`:
|
||||
passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `make pre-commit`: passed.
|
||||
- Runtime thin bridge scan: passed; notify, obs metrics, Swift, and S3
|
||||
Select no longer declare local `ecstore_compat` modules.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; changes only move import/source boundaries.
|
||||
|
||||
- Issue #660 API-140 current slice:
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; existing capacity metrics casts, HTTP atomic
|
||||
relaxed counters, and HTTP test unwrap/expect calls remain unchanged.
|
||||
|
||||
- Issue #660 API-141 current slice:
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `make pre-commit`: passed.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; only import aliases were reported by the textual
|
||||
`as` scan.
|
||||
|
||||
- Issue #660 API-142 current slice:
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `make pre-commit`: passed.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; only type/import aliases were reported by the
|
||||
textual `as` scan.
|
||||
|
||||
- Issue #660 API-143 current slice:
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `make pre-commit`: passed.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; only existing `DiskResult<Vec<String>>`
|
||||
textual matches were reported by the broad error-type scan.
|
||||
|
||||
- Issue #660 API-144 current slice:
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `make pre-commit`: passed.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; only existing `DiskResult<Vec<String>>`
|
||||
textual matches were reported by the broad error-type scan.
|
||||
|
||||
- Issue #660 API-145 current slice:
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `make pre-commit`: passed.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; only existing `DiskResult<Vec<String>>`
|
||||
textual matches were reported by the broad error-type scan.
|
||||
|
||||
- Issue #660 API-146 current slice:
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `make pre-commit`: passed.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; only existing `DiskResult<Vec<String>>`
|
||||
textual matches were reported by the broad error-type scan.
|
||||
|
||||
- Issue #660 API-147 current slice:
|
||||
- `cargo check -p rustfs-notify -p rustfs-obs -p rustfs-s3select-api -p rustfs-protocols -p rustfs-iam -p rustfs-heal -p rustfs-scanner`:
|
||||
passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `make pre-commit`: passed.
|
||||
- Runtime crate ECStore source bypass scan: passed; target runtime crate
|
||||
source paths now reference `rustfs_ecstore::api::` only inside
|
||||
`ecstore_compat.rs`.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; changes only move import/source boundaries.
|
||||
|
||||
- Issue #660 API-148 current slice:
|
||||
- `cargo check --tests -p rustfs-heal -p rustfs-scanner -p e2e_test`:
|
||||
passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `make pre-commit`: passed.
|
||||
- External test ECStore source bypass scan: passed; target test/e2e paths now
|
||||
reference `rustfs_ecstore::api::` only inside `ecstore_test_compat.rs`.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; changes only move import/source boundaries.
|
||||
|
||||
- Issue #660 API-149 current slice:
|
||||
- `cargo check --manifest-path fuzz/Cargo.toml --bins`: passed; Cargo
|
||||
refreshed the stale fuzz lockfile during verification and the generated
|
||||
lockfile change was not retained.
|
||||
- `cargo fmt --all --manifest-path fuzz/Cargo.toml`: passed.
|
||||
- `cargo fmt --all --check --manifest-path fuzz/Cargo.toml`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `make pre-commit`: passed.
|
||||
- Fuzz ECStore source bypass scan: passed; fuzz targets now reference
|
||||
`rustfs_ecstore::api::` only inside `ecstore_fuzz_compat.rs`.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; changes only move fuzz import/source boundaries.
|
||||
|
||||
- Issue #660 API-150 current slice:
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `make pre-commit`: passed.
|
||||
- Storage owner direct ECStore source scan: passed; `rustfs/src/storage/mod.rs`
|
||||
contains no direct `rustfs_ecstore::api::` source path.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; changes only move storage owner import/source
|
||||
boundaries.
|
||||
|
||||
- Issue #660 API-139 current slice:
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; existing startup-server test `expect` calls
|
||||
remain test-only and unchanged.
|
||||
|
||||
- Issue #660 API-138 current slice:
|
||||
- `cargo check -p rustfs-notify -p rustfs-s3select-api`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- Rust risk scan: no new production unwrap/expect, casts, panic/todo/unsafe,
|
||||
or error-type risks added; existing S3 Select unwrap and Notify
|
||||
`Result<String>` wrapper signatures remain unchanged.
|
||||
|
||||
- Issue #660 API-137 current slice:
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- Stacked-base freshness check against
|
||||
`origin/overtrue/arch-test-fuzz-owner-symbols`: passed.
|
||||
|
||||
- Issue #660 API-136 current slice:
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `cargo fmt --all`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `bash -n scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||
- `make pre-commit`: passed.
|
||||
- Completed runtime-owner module-alias residual scan: passed.
|
||||
- Rust risk scan: no new production unwrap/expect, panic/todo/unsafe, or
|
||||
risky behavior added; existing `Result<Vec<String>>` storage trait
|
||||
signatures remain unchanged compatibility surfaces.
|
||||
|
||||
- Issue #660 API-135 current slice:
|
||||
- `cargo check --tests -p e2e_test -p rustfs-heal -p rustfs-scanner`: passed.
|
||||
- `cargo check --manifest-path fuzz/Cargo.toml --bins`: passed.
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
#![no_main]
|
||||
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
use rustfs_ecstore::api::bucket::utils::{
|
||||
mod ecstore_fuzz_compat;
|
||||
use ecstore_fuzz_compat::{
|
||||
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use rustfs_ecstore::api::bucket as ecstore_bucket;
|
||||
use rustfs_ecstore::api::error as ecstore_error;
|
||||
|
||||
pub(crate) fn check_bucket_and_object_names(bucket: &str, object: &str) -> ecstore_error::Result<()> {
|
||||
ecstore_bucket::utils::check_bucket_and_object_names(bucket, object)
|
||||
}
|
||||
|
||||
pub(crate) fn check_list_objs_args(bucket: &str, prefix: &str, marker: &Option<String>) -> ecstore_error::Result<()> {
|
||||
ecstore_bucket::utils::check_list_objs_args(bucket, prefix, marker)
|
||||
}
|
||||
|
||||
pub(crate) fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> ecstore_error::Result<()> {
|
||||
ecstore_bucket::utils::check_object_name_for_length_and_slash(bucket, object)
|
||||
}
|
||||
|
||||
pub(crate) fn check_valid_bucket_name_strict(bucket: &str) -> ecstore_error::Result<()> {
|
||||
ecstore_bucket::utils::check_valid_bucket_name_strict(bucket)
|
||||
}
|
||||
|
||||
pub(crate) fn has_bad_path_component(object: &str) -> bool {
|
||||
ecstore_bucket::utils::has_bad_path_component(object)
|
||||
}
|
||||
|
||||
pub(crate) fn is_meta_bucketname(bucket: &str) -> bool {
|
||||
ecstore_bucket::utils::is_meta_bucketname(bucket)
|
||||
}
|
||||
|
||||
pub(crate) fn is_valid_object_prefix(object: &str) -> bool {
|
||||
ecstore_bucket::utils::is_valid_object_prefix(object)
|
||||
}
|
||||
@@ -1,9 +1,8 @@
|
||||
#![no_main]
|
||||
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
use rustfs_ecstore::api::bucket::utils::{
|
||||
check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix,
|
||||
};
|
||||
mod ecstore_fuzz_compat;
|
||||
use ecstore_fuzz_compat::{check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix};
|
||||
use rustfs_utils::path::{clean, path_join};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
|
||||
+72
-15
@@ -103,21 +103,78 @@ fn register_admin_routes(r: &mut S3Router<AdminOperation>) -> std::io::Result<()
|
||||
use std::ops::Deref;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_ecstore::api::bucket as ecstore_bucket;
|
||||
use rustfs_ecstore::api::capacity as ecstore_capacity;
|
||||
use rustfs_ecstore::api::client as ecstore_client;
|
||||
use rustfs_ecstore::api::config as ecstore_config;
|
||||
use rustfs_ecstore::api::data_usage as ecstore_data_usage;
|
||||
use rustfs_ecstore::api::disk as ecstore_disk;
|
||||
use rustfs_ecstore::api::error as ecstore_error;
|
||||
use rustfs_ecstore::api::global as ecstore_global;
|
||||
use rustfs_ecstore::api::layout as ecstore_layout;
|
||||
use rustfs_ecstore::api::metrics as ecstore_metrics;
|
||||
use rustfs_ecstore::api::notification as ecstore_notification;
|
||||
use rustfs_ecstore::api::rebalance as ecstore_rebalance;
|
||||
use rustfs_ecstore::api::rpc as ecstore_rpc;
|
||||
use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
use rustfs_ecstore::api::tier as ecstore_tier;
|
||||
mod ecstore_bucket {
|
||||
pub(crate) use crate::storage::ecstore_bucket::{
|
||||
bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning,
|
||||
versioning_sys,
|
||||
};
|
||||
}
|
||||
|
||||
mod ecstore_capacity {
|
||||
pub(crate) use crate::storage::ecstore_capacity::is_reserved_or_invalid_bucket;
|
||||
}
|
||||
|
||||
mod ecstore_client {
|
||||
pub(crate) use crate::storage::ecstore_client::admin_handler_utils;
|
||||
}
|
||||
|
||||
mod ecstore_config {
|
||||
pub(crate) use crate::storage::ecstore_config::{com, init, set_global_storage_class, storageclass};
|
||||
}
|
||||
|
||||
mod ecstore_data_usage {
|
||||
pub(crate) use crate::storage::ecstore_data_usage::load_data_usage_from_backend;
|
||||
}
|
||||
|
||||
#[allow(unused_imports)]
|
||||
mod ecstore_disk {
|
||||
pub(crate) use crate::storage::ecstore_disk::{RUSTFS_META_BUCKET, endpoint};
|
||||
}
|
||||
|
||||
mod ecstore_error {
|
||||
pub(crate) use crate::storage::ecstore_error::StorageError;
|
||||
}
|
||||
|
||||
mod ecstore_global {
|
||||
pub(crate) use crate::storage::ecstore_global::{
|
||||
GLOBAL_BOOT_TIME, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt, get_global_region,
|
||||
global_rustfs_port,
|
||||
};
|
||||
}
|
||||
|
||||
#[allow(unused_imports)]
|
||||
mod ecstore_layout {
|
||||
pub(crate) use crate::storage::ecstore_layout::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
}
|
||||
|
||||
mod ecstore_metrics {
|
||||
pub(crate) use crate::storage::ecstore_metrics::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
||||
}
|
||||
|
||||
mod ecstore_notification {
|
||||
pub(crate) use crate::storage::ecstore_notification::{NotificationSys, get_global_notification_sys};
|
||||
}
|
||||
|
||||
#[allow(unused_imports)]
|
||||
mod ecstore_rebalance {
|
||||
pub(crate) use crate::storage::ecstore_rebalance::{
|
||||
DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo,
|
||||
RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, decode_rebalance_stop_propagation_record,
|
||||
encode_rebalance_stop_propagation_record,
|
||||
};
|
||||
}
|
||||
|
||||
mod ecstore_rpc {
|
||||
pub(crate) use crate::storage::ecstore_rpc::PeerRestClient;
|
||||
}
|
||||
|
||||
mod ecstore_storage {
|
||||
pub(crate) use crate::storage::ecstore_storage::ECStore;
|
||||
}
|
||||
|
||||
mod ecstore_tier {
|
||||
pub(crate) use crate::storage::ecstore_tier::{tier, tier_admin, tier_config, tier_handlers};
|
||||
}
|
||||
|
||||
pub(crate) const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET;
|
||||
pub(crate) const STORAGE_CLASS_SUB_SYS: &str = ecstore_config::com::STORAGE_CLASS_SUB_SYS;
|
||||
|
||||
+83
-53
@@ -43,54 +43,84 @@ mod lifecycle_transition_api_test;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_ecstore::api::admin as ecstore_admin;
|
||||
use rustfs_ecstore::api::bucket as ecstore_bucket;
|
||||
use rustfs_ecstore::api::capacity as ecstore_capacity;
|
||||
use rustfs_ecstore::api::client as ecstore_client;
|
||||
use rustfs_ecstore::api::compression as ecstore_compression;
|
||||
use rustfs_ecstore::api::config as ecstore_config;
|
||||
use rustfs_ecstore::api::data_usage as ecstore_data_usage;
|
||||
use rustfs_ecstore::api::disk as ecstore_disk;
|
||||
use rustfs_ecstore::api::error as ecstore_error;
|
||||
use rustfs_ecstore::api::global as ecstore_global;
|
||||
use rustfs_ecstore::api::layout as ecstore_layout;
|
||||
use rustfs_ecstore::api::notification as ecstore_notification;
|
||||
use rustfs_ecstore::api::rio as ecstore_rio;
|
||||
use rustfs_ecstore::api::set_disk as ecstore_set_disk;
|
||||
use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
use rustfs_ecstore::api::tier as ecstore_tier;
|
||||
mod ecstore_admin {
|
||||
pub(crate) use crate::storage::ecstore_admin::get_server_info;
|
||||
}
|
||||
|
||||
mod ecstore_bucket {
|
||||
pub(crate) use crate::storage::ecstore_bucket::{
|
||||
bucket_target_sys, lifecycle, metadata, metadata_sys, object_lock, policy_sys, quota, replication, tagging, target,
|
||||
utils, versioning, versioning_sys,
|
||||
};
|
||||
}
|
||||
|
||||
mod ecstore_capacity {
|
||||
pub(crate) use crate::storage::ecstore_capacity::{
|
||||
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
||||
};
|
||||
}
|
||||
|
||||
#[allow(unused_imports)]
|
||||
mod ecstore_client {
|
||||
pub(crate) use crate::storage::ecstore_client::{object_api_utils, transition_api};
|
||||
}
|
||||
|
||||
mod ecstore_compression {
|
||||
pub(crate) use crate::storage::ecstore_compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||
}
|
||||
|
||||
mod ecstore_config {
|
||||
pub(crate) use crate::storage::ecstore_config::storageclass;
|
||||
}
|
||||
|
||||
mod ecstore_data_usage {
|
||||
pub(crate) use crate::storage::ecstore_data_usage::{
|
||||
apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory,
|
||||
record_bucket_object_write_memory,
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod ecstore_global {
|
||||
pub(crate) use crate::storage::ecstore_global::GLOBAL_TierConfigMgr;
|
||||
}
|
||||
|
||||
#[allow(unused_imports)]
|
||||
mod ecstore_tier {
|
||||
pub(crate) use crate::storage::ecstore_tier::{tier, tier_config, warm_backend};
|
||||
}
|
||||
|
||||
pub(crate) const MIN_DISK_COMPRESSIBLE_SIZE: usize = ecstore_compression::MIN_DISK_COMPRESSIBLE_SIZE;
|
||||
|
||||
pub(crate) type DiskError = ecstore_disk::error::DiskError;
|
||||
pub(crate) type DynReader = ecstore_rio::DynReader;
|
||||
pub(crate) type ECStore = ecstore_storage::ECStore;
|
||||
pub(crate) type EndpointServerPools = ecstore_layout::EndpointServerPools;
|
||||
pub(crate) type HashReader = ecstore_rio::HashReader;
|
||||
pub(crate) type NotificationSys = ecstore_notification::NotificationSys;
|
||||
pub(crate) type ObjectStoreResolver = dyn Fn() -> Option<Arc<ECStore>> + Send + Sync + 'static;
|
||||
pub(crate) type DiskError = crate::storage::DiskError;
|
||||
pub(crate) type DynReader = crate::storage::DynReader;
|
||||
pub(crate) type ECStore = crate::storage::ECStore;
|
||||
pub(crate) type EndpointServerPools = crate::storage::EndpointServerPools;
|
||||
pub(crate) type HashReader = crate::storage::HashReader;
|
||||
pub(crate) type NotificationSys = crate::storage::NotificationSys;
|
||||
pub(crate) type ObjectStoreResolver = crate::storage::ObjectStoreResolver;
|
||||
pub(crate) type ObjectInfo = <ECStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
|
||||
pub(crate) type ObjectOptions = <ECStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
|
||||
pub(crate) type PoolDecommissionInfo = ecstore_capacity::PoolDecommissionInfo;
|
||||
pub(crate) type PoolStatus = ecstore_capacity::PoolStatus;
|
||||
pub(crate) type StorageError = ecstore_error::StorageError;
|
||||
pub(crate) type StorageError = crate::storage::StorageError;
|
||||
pub(crate) type Error = StorageError;
|
||||
pub(crate) type TierConfigMgr = ecstore_tier::tier::TierConfigMgr;
|
||||
pub(crate) type WriteEncryption = ecstore_rio::WriteEncryption;
|
||||
pub(crate) type WritePlan = ecstore_rio::WritePlan;
|
||||
pub(crate) type TierConfigMgr = crate::storage::TierConfigMgr;
|
||||
pub(crate) type WriteEncryption = crate::storage::WriteEncryption;
|
||||
pub(crate) type WritePlan = crate::storage::WritePlan;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) type DecryptReader<R> = ecstore_rio::DecryptReader<R>;
|
||||
pub(crate) type DecryptReader<R> = crate::storage::DecryptReader<R>;
|
||||
#[cfg(test)]
|
||||
pub(crate) type EncryptReader<R> = ecstore_rio::EncryptReader<R>;
|
||||
pub(crate) type EncryptReader<R> = crate::storage::EncryptReader<R>;
|
||||
#[cfg(test)]
|
||||
pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint;
|
||||
pub(crate) type Endpoint = crate::storage::Endpoint;
|
||||
#[cfg(test)]
|
||||
pub(crate) type Endpoints = ecstore_layout::Endpoints;
|
||||
pub(crate) type Endpoints = crate::storage::Endpoints;
|
||||
#[cfg(test)]
|
||||
pub(crate) type HardLimitReader<R> = ecstore_rio::HardLimitReader<R>;
|
||||
pub(crate) type HardLimitReader<R> = crate::storage::HardLimitReader<R>;
|
||||
#[cfg(test)]
|
||||
pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints;
|
||||
pub(crate) type PoolEndpoints = crate::storage::PoolEndpoints;
|
||||
#[cfg(test)]
|
||||
pub(crate) type TierConfig = ecstore_tier::tier_config::TierConfig;
|
||||
#[cfg(test)]
|
||||
@@ -103,7 +133,7 @@ pub(crate) type WarmBackendGetOpts = ecstore_tier::warm_backend::WarmBackendGetO
|
||||
#[cfg(test)]
|
||||
#[allow(non_snake_case)]
|
||||
pub(crate) fn EndpointServerPools(pools: Vec<PoolEndpoints>) -> EndpointServerPools {
|
||||
ecstore_layout::EndpointServerPools::from(pools)
|
||||
crate::storage::EndpointServerPools::from(pools)
|
||||
}
|
||||
|
||||
pub(crate) trait AppObjectLockConfigExt {
|
||||
@@ -396,7 +426,7 @@ pub(crate) mod metadata_sys {
|
||||
|
||||
pub(crate) mod object_api_utils {
|
||||
pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag {
|
||||
super::ecstore_client::object_api_utils::to_s3s_etag(etag)
|
||||
crate::storage::to_s3s_etag(etag)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -565,19 +595,19 @@ pub(crate) async fn record_bucket_object_write_memory(bucket: &str, previous_cur
|
||||
}
|
||||
|
||||
pub(crate) fn is_all_buckets_not_found(errs: &[Option<DiskError>]) -> bool {
|
||||
ecstore_disk::error_reduce::is_all_buckets_not_found(errs)
|
||||
crate::storage::is_all_buckets_not_found(errs)
|
||||
}
|
||||
|
||||
pub(crate) fn is_err_bucket_not_found(err: &Error) -> bool {
|
||||
ecstore_error::is_err_bucket_not_found(err)
|
||||
crate::storage::is_err_bucket_not_found(err)
|
||||
}
|
||||
|
||||
pub(crate) fn is_err_object_not_found(err: &Error) -> bool {
|
||||
ecstore_error::is_err_object_not_found(err)
|
||||
crate::storage::is_err_object_not_found(err)
|
||||
}
|
||||
|
||||
pub(crate) fn is_err_version_not_found(err: &Error) -> bool {
|
||||
ecstore_error::is_err_version_not_found(err)
|
||||
crate::storage::is_err_version_not_found(err)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -597,57 +627,57 @@ impl std::ops::Deref for GlobalTierConfigMgrCompat {
|
||||
}
|
||||
|
||||
pub(crate) fn get_global_endpoints_opt() -> Option<EndpointServerPools> {
|
||||
ecstore_global::get_global_endpoints_opt()
|
||||
crate::storage::get_global_endpoints_opt()
|
||||
}
|
||||
|
||||
pub(crate) fn get_global_region() -> Option<s3s::region::Region> {
|
||||
ecstore_global::get_global_region()
|
||||
crate::storage::get_global_region()
|
||||
}
|
||||
|
||||
pub(crate) fn get_global_tier_config_mgr() -> Arc<tokio::sync::RwLock<TierConfigMgr>> {
|
||||
ecstore_global::get_global_tier_config_mgr()
|
||||
crate::storage::get_global_tier_config_mgr()
|
||||
}
|
||||
|
||||
pub(crate) fn new_object_layer_fn() -> Option<Arc<ECStore>> {
|
||||
ecstore_global::new_object_layer_fn()
|
||||
crate::storage::new_object_layer_fn()
|
||||
}
|
||||
|
||||
pub(crate) fn set_object_store_resolver(resolver: Arc<ObjectStoreResolver>) -> bool {
|
||||
ecstore_global::set_object_store_resolver(resolver)
|
||||
crate::storage::set_object_store_resolver(resolver)
|
||||
}
|
||||
|
||||
pub(crate) fn get_global_notification_sys() -> Option<&'static NotificationSys> {
|
||||
ecstore_notification::get_global_notification_sys()
|
||||
crate::storage::get_global_notification_sys()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn boxed_reader<R>(reader: R) -> DynReader
|
||||
where
|
||||
R: ecstore_rio::Reader + 'static,
|
||||
R: crate::storage::ecstore_rio::Reader + 'static,
|
||||
{
|
||||
ecstore_rio::boxed_reader(reader)
|
||||
crate::storage::boxed_reader(reader)
|
||||
}
|
||||
|
||||
pub(crate) fn compression_metadata_value(algorithm: rustfs_utils::CompressionAlgorithm) -> String {
|
||||
ecstore_rio::compression_metadata_value(algorithm)
|
||||
crate::storage::compression_metadata_value(algorithm)
|
||||
}
|
||||
|
||||
pub(crate) fn wrap_reader<R>(reader: R) -> DynReader
|
||||
where
|
||||
R: ecstore_rio::ReadStream + 'static,
|
||||
R: crate::storage::ecstore_rio::ReadStream + 'static,
|
||||
{
|
||||
ecstore_rio::wrap_reader(reader)
|
||||
crate::storage::wrap_reader(reader)
|
||||
}
|
||||
|
||||
pub(crate) fn get_lock_acquire_timeout() -> tokio::time::Duration {
|
||||
ecstore_set_disk::get_lock_acquire_timeout()
|
||||
crate::storage::get_lock_acquire_timeout()
|
||||
}
|
||||
|
||||
pub(crate) fn is_valid_storage_class(storage_class: &str) -> bool {
|
||||
ecstore_set_disk::is_valid_storage_class(storage_class)
|
||||
crate::storage::is_valid_storage_class(storage_class)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<(), Error> {
|
||||
ecstore_storage::init_local_disks(endpoint_pools).await
|
||||
crate::storage::init_local_disks(endpoint_pools).await
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::{ecstore_disk, ecstore_storage};
|
||||
use crate::storage::{all_local_disk, disk_drive_path, disk_endpoint};
|
||||
use rustfs_io_metrics::capacity_metrics::{
|
||||
record_capacity_cache_hit, record_capacity_cache_miss, record_capacity_cache_served, record_capacity_refresh_request,
|
||||
record_capacity_scan_mode,
|
||||
@@ -263,7 +263,7 @@ pub async fn init_capacity_management_for_local_disks() {
|
||||
"Capacity manager state changed"
|
||||
);
|
||||
|
||||
let disks = ecstore_storage::all_local_disk().await;
|
||||
let disks = all_local_disk().await;
|
||||
if disks.is_empty() {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_CAPACITY,
|
||||
@@ -286,12 +286,7 @@ pub async fn init_capacity_management_for_local_disks() {
|
||||
|
||||
let disk_refs = disks
|
||||
.iter()
|
||||
.map(|ds| {
|
||||
capacity_disk_ref(
|
||||
ecstore_disk::DiskAPI::endpoint(ds.as_ref()).to_string(),
|
||||
ecstore_disk::DiskAPI::to_string(ds.as_ref()),
|
||||
)
|
||||
})
|
||||
.map(|ds| capacity_disk_ref(disk_endpoint(ds), disk_drive_path(ds)))
|
||||
.collect();
|
||||
|
||||
info!(
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
#[allow(unsafe_op_in_unsafe_fn)]
|
||||
mod tests {
|
||||
use crate::config::{CommandResult, Config, Opt, TlsCommands};
|
||||
use crate::storage::ecstore_layout::DisksLayout;
|
||||
use crate::storage::DisksLayout;
|
||||
use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_OBS_ENDPOINT, RUSTFS_REGION};
|
||||
use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY};
|
||||
use serial_test::serial;
|
||||
@@ -262,7 +262,7 @@ mod tests {
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_volumes_and_disk_layout_parsing() {
|
||||
use crate::storage::ecstore_layout::DisksLayout;
|
||||
use crate::storage::DisksLayout;
|
||||
|
||||
// Test case 1: Single volume path
|
||||
let args = vec!["rustfs", "/data/vol1"];
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::{ecstore_bucket::quota::QuotaError, ecstore_error::StorageError};
|
||||
use crate::storage::{QuotaError, StorageError};
|
||||
use rustfs_storage_api::HTTPRangeError;
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
|
||||
|
||||
+17
-17
@@ -13,8 +13,10 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::server::ShutdownHandle;
|
||||
use crate::storage::{ecstore_bucket::metadata_sys as ecstore_metadata_sys, ecstore_global};
|
||||
use crate::storage::{process_lambda_configurations, process_queue_configurations, process_topic_configurations};
|
||||
use crate::storage::{
|
||||
get_bucket_notification_config, get_global_region, process_lambda_configurations, process_queue_configurations,
|
||||
process_topic_configurations,
|
||||
};
|
||||
use crate::{admin, config, version};
|
||||
use rustfs_config::{
|
||||
DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK,
|
||||
@@ -157,7 +159,7 @@ fn arn_to_target_id(arn_str: &str) -> Result<rustfs_targets::arn::TargetID, Targ
|
||||
/// * `buckets` - A vector of bucket names to process
|
||||
#[instrument(skip_all)]
|
||||
pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
|
||||
let global_region = ecstore_global::get_global_region();
|
||||
let global_region = get_global_region();
|
||||
let region = global_region
|
||||
.as_ref()
|
||||
.filter(|r| !r.as_str().is_empty())
|
||||
@@ -174,20 +176,18 @@ pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
|
||||
RUSTFS_REGION
|
||||
});
|
||||
for bucket in buckets.iter() {
|
||||
let has_notification_config = ecstore_metadata_sys::get_notification_config(bucket)
|
||||
.await
|
||||
.unwrap_or_else(|err| {
|
||||
warn!(
|
||||
target: "rustfs::init",
|
||||
event = "notification_config_load_failed",
|
||||
component = LOG_COMPONENT_INIT,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
bucket = %bucket,
|
||||
error = ?err,
|
||||
"Failed to load bucket notification configuration"
|
||||
);
|
||||
None
|
||||
});
|
||||
let has_notification_config = get_bucket_notification_config(bucket).await.unwrap_or_else(|err| {
|
||||
warn!(
|
||||
target: "rustfs::init",
|
||||
event = "notification_config_load_failed",
|
||||
component = LOG_COMPONENT_INIT,
|
||||
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
|
||||
bucket = %bucket,
|
||||
error = ?err,
|
||||
"Failed to load bucket notification configuration"
|
||||
);
|
||||
None
|
||||
});
|
||||
|
||||
match has_notification_config {
|
||||
Some(cfg) => {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::{ecstore_cluster, ecstore_layout::EndpointServerPools};
|
||||
use crate::storage::{EndpointServerPools, topology_snapshot_from_endpoint_pools_with_capabilities};
|
||||
use rustfs_storage_api::{
|
||||
CapabilitySnapshotError, CapabilityStatus, DiskCapabilities, MemorySamplingState, ObservabilitySnapshot,
|
||||
ObservabilitySnapshotProvider, PlatformSupport, TopologyCapabilities, TopologySnapshot, TopologySnapshotProvider,
|
||||
@@ -79,7 +79,7 @@ impl TopologySnapshotProvider for EndpointTopologySnapshotProvider {
|
||||
}
|
||||
|
||||
pub fn topology_snapshot_from_endpoint_pools(endpoint_pools: &EndpointServerPools) -> TopologySnapshot {
|
||||
ecstore_cluster::topology_snapshot_from_endpoint_pools_with_capabilities(
|
||||
topology_snapshot_from_endpoint_pools_with_capabilities(
|
||||
endpoint_pools,
|
||||
TopologyCapabilities {
|
||||
profiling: cpu_profiling_status(),
|
||||
@@ -131,10 +131,7 @@ fn cgroup_memory_status() -> CapabilityStatus {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::storage::{
|
||||
ecstore_disk::endpoint::Endpoint,
|
||||
ecstore_layout::{Endpoints, PoolEndpoints},
|
||||
};
|
||||
use crate::storage::{Endpoint, Endpoints, PoolEndpoints};
|
||||
use rustfs_storage_api::{CapabilityState, ObservabilitySnapshotProvider, TopologySnapshotProvider};
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -14,8 +14,7 @@
|
||||
|
||||
use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store};
|
||||
use crate::app::context::resolve_server_config;
|
||||
use crate::storage::StorageObjectInfo;
|
||||
use crate::storage::ecstore_event::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook};
|
||||
use crate::storage::{EventArgs as EcstoreEventArgs, StorageObjectInfo, register_event_dispatch_hook};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rustfs_notify::{EventArgs as NotifyEventArgs, NotifyObjectInfo};
|
||||
use rustfs_s3_types::EventName;
|
||||
|
||||
@@ -31,9 +31,9 @@ use crate::server::{
|
||||
},
|
||||
};
|
||||
use crate::storage;
|
||||
use crate::storage::ecstore_rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
|
||||
use crate::storage::rpc::InternodeRpcService;
|
||||
use crate::storage::tonic_service::make_server;
|
||||
use crate::storage::{TONIC_RPC_PREFIX, verify_rpc_signature};
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, Method, Request as HttpRequest, Response};
|
||||
use hyper_util::{
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::{ecstore_config, ecstore_error::Error as StorageError, ecstore_global::resolve_object_store_handle};
|
||||
use crate::storage::{Error as StorageError, read_config, resolve_object_store_handle, save_config};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
@@ -162,7 +162,7 @@ pub(crate) async fn refresh_persisted_module_switches_from_store() -> Result<Per
|
||||
return Err("storage layer not initialized".to_string());
|
||||
};
|
||||
|
||||
let (config, configured) = match ecstore_config::com::read_config(store, MODULE_SWITCH_CONFIG_PATH).await {
|
||||
let (config, configured) = match read_config(store, MODULE_SWITCH_CONFIG_PATH).await {
|
||||
Ok(data) => (
|
||||
serde_json::from_slice::<PersistedModuleSwitches>(&data)
|
||||
.map_err(|e| format!("failed to deserialize module switch config: {e}"))?,
|
||||
@@ -184,7 +184,7 @@ pub(crate) async fn save_persisted_module_switches_to_store(config: PersistedMod
|
||||
};
|
||||
|
||||
let data = serde_json::to_vec(&config).map_err(|e| format!("failed to serialize module switch config: {e}"))?;
|
||||
ecstore_config::com::save_config(store, MODULE_SWITCH_CONFIG_PATH, data)
|
||||
save_config(store, MODULE_SWITCH_CONFIG_PATH, data)
|
||||
.await
|
||||
.map_err(|e| format!("failed to save module switch config: {e}"))?;
|
||||
|
||||
|
||||
@@ -14,13 +14,12 @@
|
||||
|
||||
use crate::server::{ServiceState, ServiceStateManager};
|
||||
use crate::server::{has_path_prefix, is_table_catalog_path};
|
||||
#[cfg(test)]
|
||||
use crate::storage::ecstore_layout::{Endpoints, PoolEndpoints};
|
||||
use crate::storage::{
|
||||
ecstore_disk::endpoint::Endpoint,
|
||||
ecstore_global::{get_global_endpoints_opt, get_global_lock_clients, is_dist_erasure, resolve_object_store_handle},
|
||||
ecstore_layout::EndpointServerPools,
|
||||
Endpoint, EndpointServerPools, get_global_endpoints_opt, get_global_lock_clients, is_dist_erasure,
|
||||
resolve_object_store_handle,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::storage::{Endpoints, PoolEndpoints};
|
||||
use bytes::Bytes;
|
||||
use http::{Request as HttpRequest, Response, StatusCode};
|
||||
use http_body::Body;
|
||||
|
||||
@@ -13,8 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::{
|
||||
ECStore,
|
||||
ecstore_bucket::{metadata_sys as ecstore_metadata_sys, migration as ecstore_migration, replication as ecstore_replication},
|
||||
ECStore, get_global_replication_pool, init_bucket_metadata_sys, try_migrate_bucket_metadata, try_migrate_iam_config,
|
||||
};
|
||||
use rustfs_storage_api::{BucketOperations, BucketOptions};
|
||||
use std::{
|
||||
@@ -34,9 +33,9 @@ pub(crate) async fn init_embedded_bucket_metadata_runtime(store: Arc<ECStore>) -
|
||||
|
||||
let buckets: Vec<String> = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
|
||||
ecstore_migration::try_migrate_bucket_metadata(store.clone()).await;
|
||||
ecstore_metadata_sys::init_bucket_metadata_sys(store.clone(), buckets.clone()).await;
|
||||
ecstore_migration::try_migrate_iam_config(store).await;
|
||||
try_migrate_bucket_metadata(store.clone()).await;
|
||||
init_bucket_metadata_sys(store.clone(), buckets.clone()).await;
|
||||
try_migrate_iam_config(store).await;
|
||||
|
||||
Ok(buckets)
|
||||
}
|
||||
@@ -52,14 +51,14 @@ pub(crate) async fn init_bucket_metadata_runtime(store: Arc<ECStore>, ctx: Cance
|
||||
|
||||
let buckets: Vec<String> = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
|
||||
ecstore_migration::try_migrate_bucket_metadata(store.clone()).await;
|
||||
try_migrate_bucket_metadata(store.clone()).await;
|
||||
|
||||
if let Some(pool) = ecstore_replication::get_global_replication_pool() {
|
||||
if let Some(pool) = get_global_replication_pool() {
|
||||
pool.init_resync(ctx, buckets.clone()).await?;
|
||||
}
|
||||
|
||||
ecstore_migration::try_migrate_iam_config(store.clone()).await;
|
||||
ecstore_metadata_sys::init_bucket_metadata_sys(store, buckets.clone()).await;
|
||||
try_migrate_iam_config(store.clone()).await;
|
||||
init_bucket_metadata_sys(store, buckets.clone()).await;
|
||||
|
||||
Ok(buckets)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::ecstore_layout::EndpointServerPools;
|
||||
use crate::storage::EndpointServerPools;
|
||||
use rustfs_config::{
|
||||
DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, ENV_RUSTFS_UNSUPPORTED_FS_POLICY, RUSTFS_UNSUPPORTED_FS_POLICY_FAIL,
|
||||
RUSTFS_UNSUPPORTED_FS_POLICY_WARN,
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::init::add_bucket_notification_configuration;
|
||||
use crate::storage::{ecstore_error, ecstore_layout::EndpointServerPools, ecstore_notification};
|
||||
use crate::storage::{EndpointServerPools, Result as StorageResult, new_global_notification_sys};
|
||||
use std::{
|
||||
future::Future,
|
||||
io::{Error, Result},
|
||||
@@ -50,14 +50,14 @@ pub(crate) async fn init_notification_runtime(endpoint_pools: EndpointServerPool
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn init_notification_system(endpoint_pools: EndpointServerPools) -> ecstore_error::Result<()> {
|
||||
init_notification_system_with(|| ecstore_notification::new_global_notification_sys(endpoint_pools)).await
|
||||
pub(crate) async fn init_notification_system(endpoint_pools: EndpointServerPools) -> StorageResult<()> {
|
||||
init_notification_system_with(|| new_global_notification_sys(endpoint_pools)).await
|
||||
}
|
||||
|
||||
async fn init_notification_system_with<InitFn, InitFuture>(init_notification: InitFn) -> ecstore_error::Result<()>
|
||||
async fn init_notification_system_with<InitFn, InitFuture>(init_notification: InitFn) -> StorageResult<()>
|
||||
where
|
||||
InitFn: FnOnce() -> InitFuture,
|
||||
InitFuture: Future<Output = ecstore_error::Result<()>>,
|
||||
InitFuture: Future<Output = StorageResult<()>>,
|
||||
{
|
||||
init_notification().await
|
||||
}
|
||||
@@ -76,7 +76,7 @@ fn log_embedded_optional_service_skipped(service: &str, err: impl std::fmt::Disp
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::init_notification_system_with;
|
||||
use crate::storage::ecstore_error::Error as EcstoreError;
|
||||
use crate::storage::Error as EcstoreError;
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_system_returns_source_error() {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::ecstore_global::{set_global_region, set_global_rustfs_port};
|
||||
use crate::storage::{set_global_region, set_global_rustfs_port};
|
||||
use crate::{
|
||||
capacity::capacity_integration::init_capacity_management,
|
||||
config::Config,
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::{ECStore, ecstore_layout::EndpointServerPools};
|
||||
use crate::storage::{ECStore, EndpointServerPools};
|
||||
use crate::{
|
||||
config::Config,
|
||||
init::{init_buffer_profile_system, init_kms_system},
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::ecstore_global::shutdown_background_services;
|
||||
use crate::storage::shutdown_background_services;
|
||||
use crate::{
|
||||
server::{ServiceState, ServiceStateManager, ShutdownHandle, ShutdownSignal, shutdown_event_notifier, stop_audit_system},
|
||||
startup_optional_runtime_sidecars::{
|
||||
|
||||
@@ -13,10 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
|
||||
use crate::storage::ECStore;
|
||||
use crate::storage::{
|
||||
ecstore_bucket::replication as ecstore_replication, ecstore_config, ecstore_global, ecstore_layout::EndpointServerPools,
|
||||
ecstore_storage,
|
||||
ECStore, EndpointServerPools, init_background_replication, init_ecstore_config, init_global_config_sys, init_local_disks,
|
||||
init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints, try_migrate_server_config, update_erasure_type,
|
||||
};
|
||||
use rustfs_common::{GlobalReadiness, SystemStage};
|
||||
use std::{
|
||||
@@ -71,8 +70,8 @@ pub(crate) async fn init_startup_storage_foundation(server_address: &str, volume
|
||||
.map_err(Error::other)?;
|
||||
enforce_unsupported_fs_policy(&endpoint_pools)?;
|
||||
|
||||
ecstore_global::set_global_endpoints(endpoint_pools.as_ref().clone());
|
||||
ecstore_global::update_erasure_type(setup_type).await;
|
||||
set_global_endpoints(endpoint_pools.as_ref().clone());
|
||||
update_erasure_type(setup_type).await;
|
||||
|
||||
debug!(
|
||||
target: "rustfs::main::run",
|
||||
@@ -83,7 +82,7 @@ pub(crate) async fn init_startup_storage_foundation(server_address: &str, volume
|
||||
state = "starting",
|
||||
"starting local disk initialization"
|
||||
);
|
||||
ecstore_storage::init_local_disks(endpoint_pools.clone())
|
||||
init_local_disks(endpoint_pools.clone())
|
||||
.await
|
||||
.inspect_err(|err| {
|
||||
error!(
|
||||
@@ -98,8 +97,8 @@ pub(crate) async fn init_startup_storage_foundation(server_address: &str, volume
|
||||
);
|
||||
})
|
||||
.map_err(Error::other)?;
|
||||
ecstore_storage::prewarm_local_disk_id_map().await;
|
||||
ecstore_storage::init_lock_clients(endpoint_pools.clone());
|
||||
prewarm_local_disk_id_map().await;
|
||||
init_lock_clients(endpoint_pools.clone());
|
||||
|
||||
log_storage_pool_layout(&endpoint_pools);
|
||||
|
||||
@@ -115,13 +114,13 @@ pub(crate) async fn init_embedded_startup_storage_foundation(
|
||||
.map_err(|err| Error::other(format!("endpoints: {err}")))?;
|
||||
enforce_unsupported_fs_policy(&endpoint_pools).map_err(|err| Error::other(format!("unsupported fs guard: {err}")))?;
|
||||
|
||||
ecstore_global::set_global_endpoints(endpoint_pools.as_ref().clone());
|
||||
ecstore_global::update_erasure_type(setup_type).await;
|
||||
set_global_endpoints(endpoint_pools.as_ref().clone());
|
||||
update_erasure_type(setup_type).await;
|
||||
|
||||
ecstore_storage::init_local_disks(endpoint_pools.clone())
|
||||
init_local_disks(endpoint_pools.clone())
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("local disks: {err}")))?;
|
||||
ecstore_storage::init_lock_clients(endpoint_pools.clone());
|
||||
init_lock_clients(endpoint_pools.clone());
|
||||
|
||||
Ok(endpoint_pools)
|
||||
}
|
||||
@@ -159,7 +158,7 @@ pub(crate) async fn init_startup_storage_runtime(
|
||||
|
||||
init_startup_storage_global_config(store.clone()).await?;
|
||||
readiness.mark_stage(SystemStage::StorageReady);
|
||||
ecstore_replication::init_background_replication(store.clone()).await;
|
||||
init_background_replication(store.clone()).await;
|
||||
|
||||
Ok(StartupStorageRuntime {
|
||||
store,
|
||||
@@ -190,17 +189,17 @@ pub(crate) async fn init_embedded_startup_storage_runtime(
|
||||
|
||||
init_embedded_startup_storage_global_config(store.clone()).await?;
|
||||
readiness.mark_stage(SystemStage::StorageReady);
|
||||
ecstore_replication::init_background_replication(store.clone()).await;
|
||||
init_background_replication(store.clone()).await;
|
||||
|
||||
Ok(StartupStorageRuntime { store, shutdown_token })
|
||||
}
|
||||
|
||||
async fn init_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> {
|
||||
ecstore_config::init();
|
||||
ecstore_config::try_migrate_server_config(store.clone()).await;
|
||||
init_ecstore_config();
|
||||
try_migrate_server_config(store.clone()).await;
|
||||
|
||||
let mut retry_count = 0;
|
||||
while let Err(e) = ecstore_config::init_global_config_sys(store.clone()).await {
|
||||
while let Err(e) = init_global_config_sys(store.clone()).await {
|
||||
let next_retry_count = retry_count + 1;
|
||||
error!(
|
||||
target: "rustfs::main::run",
|
||||
@@ -225,11 +224,11 @@ async fn init_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> {
|
||||
}
|
||||
|
||||
async fn init_embedded_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> {
|
||||
ecstore_config::init();
|
||||
ecstore_config::try_migrate_server_config(store.clone()).await;
|
||||
init_ecstore_config();
|
||||
try_migrate_server_config(store.clone()).await;
|
||||
|
||||
let mut retry = 0;
|
||||
while let Err(err) = ecstore_config::init_global_config_sys(store.clone()).await {
|
||||
while let Err(err) = init_global_config_sys(store.clone()).await {
|
||||
retry += 1;
|
||||
if retry > GLOBAL_CONFIG_INIT_MAX_RETRIES {
|
||||
return Err(Error::other(format!(
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod ecstore_admin {
|
||||
pub(crate) use rustfs_ecstore::api::admin::{get_local_server_property, get_server_info};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_bucket {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::{
|
||||
bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, migration, object_lock, policy_sys, replication,
|
||||
tagging, target, utils,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::bucket::{quota, versioning, versioning_sys};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_capacity {
|
||||
pub(crate) use rustfs_ecstore::api::capacity::{
|
||||
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
||||
is_reserved_or_invalid_bucket,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_client {
|
||||
pub(crate) use rustfs_ecstore::api::client::{admin_handler_utils, object_api_utils, transition_api};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_compression {
|
||||
pub(crate) use rustfs_ecstore::api::compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_cluster {
|
||||
pub(crate) use rustfs_ecstore::api::cluster::topology_snapshot_from_endpoint_pools_with_capabilities;
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_config {
|
||||
pub(crate) use rustfs_ecstore::api::config::{
|
||||
com, init, init_global_config_sys, set_global_storage_class, storageclass, try_migrate_server_config,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_data_usage {
|
||||
pub(crate) use rustfs_ecstore::api::data_usage::{
|
||||
apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory,
|
||||
record_bucket_object_write_memory,
|
||||
};
|
||||
}
|
||||
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) mod ecstore_disk {
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskStore, FileInfoVersions, FileReader, FileWriter,
|
||||
RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo,
|
||||
WalkDirOptions,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{endpoint, error, error_reduce};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_error {
|
||||
pub(crate) use rustfs_ecstore::api::error::{
|
||||
Error, Result, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_event {
|
||||
pub(crate) use rustfs_ecstore::api::event::{EventArgs, register_event_dispatch_hook};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_global {
|
||||
pub(crate) use rustfs_ecstore::api::global::{
|
||||
GLOBAL_BOOT_TIME, GLOBAL_TierConfigMgr, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt,
|
||||
get_global_lock_client, get_global_lock_clients, get_global_region, get_global_tier_config_mgr, global_rustfs_port,
|
||||
is_dist_erasure, new_object_layer_fn, resolve_object_store_handle, set_global_endpoints, set_global_region,
|
||||
set_global_rustfs_port, set_object_store_resolver, shutdown_background_services, update_erasure_type,
|
||||
};
|
||||
}
|
||||
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) mod ecstore_layout {
|
||||
pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools, Endpoints, PoolEndpoints, SetupType};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_metrics {
|
||||
pub(crate) use rustfs_ecstore::api::metrics::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
||||
}
|
||||
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) mod ecstore_notification {
|
||||
pub(crate) use rustfs_ecstore::api::notification::{
|
||||
NotificationSys, get_global_notification_sys, new_global_notification_sys,
|
||||
};
|
||||
}
|
||||
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) mod ecstore_rebalance {
|
||||
pub(crate) use rustfs_ecstore::api::rebalance::{
|
||||
DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo,
|
||||
RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, decode_rebalance_stop_propagation_record,
|
||||
encode_rebalance_stop_propagation_record,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_rio {
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rio::{DecryptReader, EncryptReader, HardLimitReader, Reader, boxed_reader};
|
||||
pub(crate) use rustfs_ecstore::api::rio::{
|
||||
DynReader, HashReader, ReadStream, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_rpc {
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, verify_rpc_signature,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_set_disk {
|
||||
pub(crate) use rustfs_ecstore::api::set_disk::{DEFAULT_READ_BUFFER_SIZE, get_lock_acquire_timeout, is_valid_storage_class};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_storage {
|
||||
pub(crate) use rustfs_ecstore::api::storage::{
|
||||
ECStore, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_lock_clients,
|
||||
prewarm_local_disk_id_map,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod ecstore_tier {
|
||||
pub(crate) use rustfs_ecstore::api::tier::tier::TierConfigMgr;
|
||||
pub(crate) use rustfs_ecstore::api::tier::{tier, tier_admin, tier_config, tier_handlers, warm_backend};
|
||||
}
|
||||
+223
-32
@@ -54,42 +54,19 @@ pub(crate) use sse::{
|
||||
validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
|
||||
};
|
||||
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::admin as ecstore_admin;
|
||||
pub(crate) use rustfs_ecstore::api::bucket as ecstore_bucket;
|
||||
pub(crate) use rustfs_ecstore::api::client as ecstore_client;
|
||||
pub(crate) use rustfs_ecstore::api::cluster as ecstore_cluster;
|
||||
pub(crate) use rustfs_ecstore::api::config as ecstore_config;
|
||||
pub(crate) use rustfs_ecstore::api::disk as ecstore_disk;
|
||||
pub(crate) use rustfs_ecstore::api::error as ecstore_error;
|
||||
pub(crate) use rustfs_ecstore::api::event as ecstore_event;
|
||||
pub(crate) use rustfs_ecstore::api::global as ecstore_global;
|
||||
pub(crate) use rustfs_ecstore::api::layout as ecstore_layout;
|
||||
pub(crate) use rustfs_ecstore::api::metrics as ecstore_metrics;
|
||||
pub(crate) use rustfs_ecstore::api::notification as ecstore_notification;
|
||||
pub(crate) use rustfs_ecstore::api::rio as ecstore_rio;
|
||||
pub(crate) use rustfs_ecstore::api::rpc as ecstore_rpc;
|
||||
pub(crate) use rustfs_ecstore::api::set_disk as ecstore_set_disk;
|
||||
pub(crate) use rustfs_ecstore::api::storage as ecstore_storage;
|
||||
pub(crate) mod ecstore_compat;
|
||||
pub(crate) use ecstore_compat::*;
|
||||
|
||||
pub(crate) const BUCKET_ACCELERATE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_ACCELERATE_CONFIG;
|
||||
pub(crate) const BUCKET_LOGGING_CONFIG: &str = ecstore_bucket::metadata::BUCKET_LOGGING_CONFIG;
|
||||
pub(crate) const BUCKET_REQUEST_PAYMENT_CONFIG: &str = ecstore_bucket::metadata::BUCKET_REQUEST_PAYMENT_CONFIG;
|
||||
pub(crate) const BUCKET_TABLE_CATALOG_META_PREFIX: &str = ecstore_bucket::metadata::BUCKET_TABLE_CATALOG_META_PREFIX;
|
||||
pub(crate) const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str =
|
||||
ecstore_bucket::metadata::BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX;
|
||||
pub(crate) const BUCKET_TABLE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_TABLE_CONFIG;
|
||||
pub(crate) const BUCKET_TABLE_RESERVED_PREFIX: &str = ecstore_bucket::metadata::BUCKET_TABLE_RESERVED_PREFIX;
|
||||
pub(crate) const BUCKET_VERSIONING_CONFIG: &str = ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG;
|
||||
pub(crate) const BUCKET_WEBSITE_CONFIG: &str = ecstore_bucket::metadata::BUCKET_WEBSITE_CONFIG;
|
||||
pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = ecstore_set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
@@ -98,6 +75,8 @@ pub(crate) const PEER_RESTSIGNAL: &str = ecstore_rpc::PEER_RESTSIGNAL;
|
||||
pub(crate) const PEER_RESTSUB_SYS: &str = ecstore_rpc::PEER_RESTSUB_SYS;
|
||||
pub(crate) const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = ecstore_rpc::SERVICE_SIGNAL_REFRESH_CONFIG;
|
||||
pub(crate) const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = ecstore_rpc::SERVICE_SIGNAL_RELOAD_DYNAMIC;
|
||||
pub(crate) const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET;
|
||||
pub(crate) const TONIC_RPC_PREFIX: &str = ecstore_rpc::TONIC_RPC_PREFIX;
|
||||
#[cfg(test)]
|
||||
pub(crate) const STORAGE_CLASS_SUB_SYS: &str = ecstore_config::com::STORAGE_CLASS_SUB_SYS;
|
||||
|
||||
@@ -113,32 +92,161 @@ pub(crate) type DiskInfo = ecstore_disk::DiskInfo;
|
||||
pub(crate) type DiskInfoOptions = ecstore_disk::DiskInfoOptions;
|
||||
pub(crate) type DiskResult<T> = ecstore_disk::error::Result<T>;
|
||||
pub(crate) type DiskStore = ecstore_disk::DiskStore;
|
||||
#[cfg(test)]
|
||||
pub(crate) type DisksLayout = ecstore_layout::DisksLayout;
|
||||
pub(crate) type DynReplicationPool = ecstore_bucket::replication::DynReplicationPool;
|
||||
pub(crate) type DynReader = ecstore_rio::DynReader;
|
||||
pub(crate) type ECStore = ecstore_storage::ECStore;
|
||||
pub(crate) type Endpoint = ecstore_disk::endpoint::Endpoint;
|
||||
#[cfg(test)]
|
||||
pub(crate) type Endpoints = ecstore_layout::Endpoints;
|
||||
pub(crate) type EndpointServerPools = ecstore_layout::EndpointServerPools;
|
||||
pub(crate) type EventArgs = ecstore_event::EventArgs;
|
||||
pub(crate) type FileInfoVersions = ecstore_disk::FileInfoVersions;
|
||||
pub(crate) type FileReader = ecstore_disk::FileReader;
|
||||
pub(crate) type FileWriter = ecstore_disk::FileWriter;
|
||||
pub(crate) type HashReader = ecstore_rio::HashReader;
|
||||
pub(crate) type LocalPeerS3Client = ecstore_rpc::LocalPeerS3Client;
|
||||
pub(crate) type MetricType = ecstore_metrics::MetricType;
|
||||
pub(crate) type ObjectPartInfo = rustfs_filemeta::ObjectPartInfo;
|
||||
pub(crate) type ObjectLockBlockReason = ecstore_bucket::object_lock::objectlock_sys::ObjectLockBlockReason;
|
||||
pub(crate) type ObjectStoreResolver = dyn Fn() -> Option<Arc<ECStore>> + Send + Sync + 'static;
|
||||
pub(crate) type PolicySys = ecstore_bucket::policy_sys::PolicySys;
|
||||
pub(crate) type PoolEndpoints = ecstore_layout::PoolEndpoints;
|
||||
pub(crate) type QuotaError = ecstore_bucket::quota::QuotaError;
|
||||
pub(crate) type RawFileInfo = rustfs_filemeta::RawFileInfo;
|
||||
pub(crate) type ReadMultipleReq = ecstore_disk::ReadMultipleReq;
|
||||
pub(crate) type ReadMultipleResp = ecstore_disk::ReadMultipleResp;
|
||||
pub(crate) type ReadOptions = ecstore_disk::ReadOptions;
|
||||
pub(crate) type RenameDataResp = ecstore_disk::RenameDataResp;
|
||||
pub(crate) type SetupType = ecstore_layout::SetupType;
|
||||
pub(crate) type StorageError = ecstore_error::StorageError;
|
||||
pub(crate) type Error = StorageError;
|
||||
pub(crate) type Result<T> = core::result::Result<T, Error>;
|
||||
pub(crate) type TierConfigMgr = ecstore_tier::TierConfigMgr;
|
||||
pub(crate) type Error = ecstore_error::Error;
|
||||
pub(crate) type Result<T> = ecstore_error::Result<T>;
|
||||
pub(crate) type UpdateMetadataOpts = ecstore_disk::UpdateMetadataOpts;
|
||||
pub(crate) type VolumeInfo = ecstore_disk::VolumeInfo;
|
||||
pub(crate) type WalkDirOptions = ecstore_disk::WalkDirOptions;
|
||||
pub(crate) type WriteEncryption = ecstore_rio::WriteEncryption;
|
||||
pub(crate) type WritePlan = ecstore_rio::WritePlan;
|
||||
#[cfg(test)]
|
||||
pub(crate) type DecryptReader<R> = ecstore_rio::DecryptReader<R>;
|
||||
#[cfg(test)]
|
||||
pub(crate) type EncryptReader<R> = ecstore_rio::EncryptReader<R>;
|
||||
#[cfg(test)]
|
||||
pub(crate) type HardLimitReader<R> = ecstore_rio::HardLimitReader<R>;
|
||||
pub(crate) type NotificationSys = ecstore_notification::NotificationSys;
|
||||
|
||||
pub(crate) async fn get_local_server_property() -> rustfs_madmin::ServerProperties {
|
||||
ecstore_admin::get_local_server_property().await
|
||||
}
|
||||
|
||||
pub(crate) async fn init_background_replication(store: Arc<ECStore>) {
|
||||
ecstore_bucket::replication::init_background_replication(store).await;
|
||||
}
|
||||
|
||||
pub(crate) async fn all_local_disk() -> Vec<DiskStore> {
|
||||
ecstore_storage::all_local_disk().await
|
||||
}
|
||||
|
||||
pub(crate) async fn get_bucket_notification_config(bucket: &str) -> Result<Option<s3s::dto::NotificationConfiguration>> {
|
||||
ecstore_bucket::metadata_sys::get_notification_config(bucket).await
|
||||
}
|
||||
|
||||
pub(crate) async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
|
||||
ecstore_bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await;
|
||||
}
|
||||
|
||||
pub(crate) fn bucket_metadata_runtime_initialized() -> bool {
|
||||
ecstore_bucket::metadata_sys::get_global_bucket_metadata_sys().is_some()
|
||||
}
|
||||
|
||||
pub(crate) fn disk_drive_path(disk: &DiskStore) -> String {
|
||||
ecstore_disk::DiskAPI::to_string(disk.as_ref())
|
||||
}
|
||||
|
||||
pub(crate) fn disk_endpoint(disk: &DiskStore) -> String {
|
||||
ecstore_disk::DiskAPI::endpoint(disk.as_ref()).to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn get_global_replication_pool() -> Option<Arc<DynReplicationPool>> {
|
||||
ecstore_bucket::replication::get_global_replication_pool()
|
||||
}
|
||||
|
||||
pub(crate) async fn try_migrate_bucket_metadata(store: Arc<ECStore>) {
|
||||
ecstore_bucket::migration::try_migrate_bucket_metadata(store).await;
|
||||
}
|
||||
|
||||
pub(crate) async fn try_migrate_iam_config(store: Arc<ECStore>) {
|
||||
ecstore_bucket::migration::try_migrate_iam_config(store).await;
|
||||
}
|
||||
|
||||
pub(crate) fn init_ecstore_config() {
|
||||
ecstore_config::init();
|
||||
}
|
||||
|
||||
pub(crate) async fn init_global_config_sys(store: Arc<ECStore>) -> Result<()> {
|
||||
ecstore_config::init_global_config_sys(store).await
|
||||
}
|
||||
|
||||
pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<()> {
|
||||
ecstore_storage::init_local_disks(endpoint_pools).await
|
||||
}
|
||||
|
||||
pub(crate) fn init_lock_clients(endpoint_pools: EndpointServerPools) {
|
||||
ecstore_storage::init_lock_clients(endpoint_pools);
|
||||
}
|
||||
|
||||
pub(crate) async fn new_global_notification_sys(endpoint_pools: EndpointServerPools) -> Result<()> {
|
||||
ecstore_notification::new_global_notification_sys(endpoint_pools).await
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config(api: Arc<ECStore>, file: &str) -> Result<Vec<u8>> {
|
||||
ecstore_config::com::read_config(api, file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn prewarm_local_disk_id_map() {
|
||||
ecstore_storage::prewarm_local_disk_id_map().await;
|
||||
}
|
||||
|
||||
pub(crate) fn replication_queue_current_count() -> Option<i64> {
|
||||
ecstore_bucket::replication::GLOBAL_REPLICATION_STATS.get().and_then(|stats| {
|
||||
stats
|
||||
.q_cache
|
||||
.try_lock()
|
||||
.ok()
|
||||
.map(|cache| cache.sr_queue_stats.curr.get_current_count())
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn save_config(api: Arc<ECStore>, file: &str, data: Vec<u8>) -> Result<()> {
|
||||
ecstore_config::com::save_config(api, file, data).await
|
||||
}
|
||||
|
||||
pub(crate) fn shutdown_background_services() {
|
||||
ecstore_global::shutdown_background_services();
|
||||
}
|
||||
|
||||
pub(crate) fn set_global_endpoints(endpoints: Vec<PoolEndpoints>) {
|
||||
ecstore_global::set_global_endpoints(endpoints);
|
||||
}
|
||||
|
||||
pub(crate) fn set_global_region(region: s3s::region::Region) {
|
||||
ecstore_global::set_global_region(region);
|
||||
}
|
||||
|
||||
pub(crate) fn set_global_rustfs_port(value: u16) {
|
||||
ecstore_global::set_global_rustfs_port(value);
|
||||
}
|
||||
|
||||
pub(crate) async fn try_migrate_server_config(store: Arc<ECStore>) {
|
||||
ecstore_config::try_migrate_server_config(store).await;
|
||||
}
|
||||
|
||||
pub(crate) async fn update_erasure_type(setup_type: SetupType) {
|
||||
ecstore_global::update_erasure_type(setup_type).await;
|
||||
}
|
||||
|
||||
pub(crate) trait StorageDiskRpcExt {
|
||||
async fn disk_info(&self, opts: &DiskInfoOptions) -> DiskResult<DiskInfo>;
|
||||
async fn delete_volume(&self, volume: &str) -> DiskResult<()>;
|
||||
@@ -562,14 +670,47 @@ pub(crate) fn is_err_version_not_found(err: &Error) -> bool {
|
||||
ecstore_error::is_err_version_not_found(err)
|
||||
}
|
||||
|
||||
pub(crate) fn is_all_buckets_not_found(errs: &[Option<DiskError>]) -> bool {
|
||||
ecstore_disk::error_reduce::is_all_buckets_not_found(errs)
|
||||
}
|
||||
|
||||
pub(crate) fn get_global_lock_client() -> Option<Arc<dyn rustfs_lock::client::LockClient>> {
|
||||
ecstore_global::get_global_lock_client()
|
||||
}
|
||||
|
||||
pub(crate) fn get_global_lock_clients()
|
||||
-> Option<&'static std::collections::HashMap<String, Arc<dyn rustfs_lock::client::LockClient>>> {
|
||||
ecstore_global::get_global_lock_clients()
|
||||
}
|
||||
|
||||
pub(crate) fn get_global_endpoints_opt() -> Option<EndpointServerPools> {
|
||||
ecstore_global::get_global_endpoints_opt()
|
||||
}
|
||||
|
||||
pub(crate) fn get_global_region() -> Option<s3s::region::Region> {
|
||||
ecstore_global::get_global_region()
|
||||
}
|
||||
|
||||
pub(crate) fn get_global_tier_config_mgr() -> Arc<tokio::sync::RwLock<TierConfigMgr>> {
|
||||
ecstore_global::get_global_tier_config_mgr()
|
||||
}
|
||||
|
||||
pub(crate) fn new_object_layer_fn() -> Option<Arc<ECStore>> {
|
||||
ecstore_global::new_object_layer_fn()
|
||||
}
|
||||
|
||||
pub(crate) fn set_object_store_resolver(resolver: Arc<ObjectStoreResolver>) -> bool {
|
||||
ecstore_global::set_object_store_resolver(resolver)
|
||||
}
|
||||
|
||||
pub(crate) fn get_global_notification_sys() -> Option<&'static NotificationSys> {
|
||||
ecstore_notification::get_global_notification_sys()
|
||||
}
|
||||
|
||||
pub(crate) async fn is_dist_erasure() -> bool {
|
||||
ecstore_global::is_dist_erasure().await
|
||||
}
|
||||
|
||||
pub(crate) fn resolve_object_store_handle() -> Option<Arc<ECStore>> {
|
||||
ecstore_global::resolve_object_store_handle()
|
||||
}
|
||||
@@ -585,6 +726,56 @@ pub(crate) fn verify_rpc_signature(url: &str, method: &http::Method, headers: &h
|
||||
ecstore_rpc::verify_rpc_signature(url, method, headers)
|
||||
}
|
||||
|
||||
pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag {
|
||||
ecstore_client::object_api_utils::to_s3s_etag(etag)
|
||||
}
|
||||
|
||||
pub(crate) fn table_catalog_path_hash(value: &str) -> String {
|
||||
ecstore_bucket::metadata::table_catalog_path_hash(value)
|
||||
}
|
||||
|
||||
pub(crate) fn get_lock_acquire_timeout() -> std::time::Duration {
|
||||
ecstore_set_disk::get_lock_acquire_timeout()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn boxed_reader<R>(reader: R) -> DynReader
|
||||
where
|
||||
R: ecstore_rio::Reader + 'static,
|
||||
{
|
||||
ecstore_rio::boxed_reader(reader)
|
||||
}
|
||||
|
||||
pub(crate) fn compression_metadata_value(algorithm: rustfs_utils::CompressionAlgorithm) -> String {
|
||||
ecstore_rio::compression_metadata_value(algorithm)
|
||||
}
|
||||
|
||||
pub(crate) fn wrap_reader<R>(reader: R) -> DynReader
|
||||
where
|
||||
R: ecstore_rio::ReadStream + 'static,
|
||||
{
|
||||
ecstore_rio::wrap_reader(reader)
|
||||
}
|
||||
|
||||
pub(crate) fn is_valid_storage_class(storage_class: &str) -> bool {
|
||||
ecstore_set_disk::is_valid_storage_class(storage_class)
|
||||
}
|
||||
|
||||
pub(crate) fn register_event_dispatch_hook<F>(hook: F) -> bool
|
||||
where
|
||||
F: Fn(EventArgs) + Send + Sync + 'static,
|
||||
{
|
||||
ecstore_event::register_event_dispatch_hook(hook)
|
||||
}
|
||||
|
||||
pub(crate) fn topology_snapshot_from_endpoint_pools_with_capabilities(
|
||||
endpoint_pools: &EndpointServerPools,
|
||||
capabilities: rustfs_storage_api::TopologyCapabilities,
|
||||
disk_capabilities: rustfs_storage_api::DiskCapabilities,
|
||||
) -> rustfs_storage_api::TopologySnapshot {
|
||||
ecstore_cluster::topology_snapshot_from_endpoint_pools_with_capabilities(endpoint_pools, capabilities, disk_capabilities)
|
||||
}
|
||||
|
||||
pub(crate) async fn reload_transition_tier_config(api: Arc<ECStore>) -> std::io::Result<()> {
|
||||
ecstore_global::GLOBAL_TierConfigMgr.write().await.reload(api).await
|
||||
}
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::ecstore_client::object_api_utils::to_s3s_etag;
|
||||
use crate::storage::s3_api::common::rustfs_owner;
|
||||
use crate::storage::to_s3s_etag;
|
||||
use percent_encoding::percent_decode_str;
|
||||
use rustfs_storage_api::{
|
||||
BucketInfo, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::ecstore_client::object_api_utils::to_s3s_etag;
|
||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||
use crate::storage::to_s3s_etag;
|
||||
use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo};
|
||||
use s3s::dto::{CommonPrefix, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp};
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
@@ -195,8 +195,8 @@ mod tests {
|
||||
MAX_MULTIPART_UPLOADS_LIST, build_list_multipart_uploads_output, build_list_parts_output,
|
||||
parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
|
||||
};
|
||||
use crate::storage::ecstore_client::object_api_utils::to_s3s_etag;
|
||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||
use crate::storage::to_s3s_etag;
|
||||
use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo};
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::Timestamp;
|
||||
|
||||
+22
-29
@@ -28,8 +28,9 @@ use std::{
|
||||
};
|
||||
|
||||
use crate::storage::{
|
||||
ecstore_bucket::{metadata as ecstore_metadata, metadata_sys as ecstore_metadata_sys},
|
||||
ecstore_disk, ecstore_error, ecstore_set_disk,
|
||||
BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG,
|
||||
BUCKET_TABLE_RESERVED_PREFIX, Error as EcstoreError, RUSTFS_META_BUCKET, StorageError, get_bucket_metadata,
|
||||
get_lock_acquire_timeout, table_catalog_path_hash,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use datafusion::{
|
||||
@@ -55,8 +56,7 @@ use crate::storage::{
|
||||
StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader,
|
||||
};
|
||||
|
||||
pub(crate) const TABLE_BUCKET_MARKER_CONFIG: &str = ecstore_metadata::BUCKET_TABLE_CONFIG;
|
||||
const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET;
|
||||
pub(crate) const TABLE_BUCKET_MARKER_CONFIG: &str = BUCKET_TABLE_CONFIG;
|
||||
pub(crate) const RESERVED_CATALOG_OBJECT_MESSAGE: &str = "Object key is reserved for the table catalog";
|
||||
pub(crate) const TABLE_BUCKET_CATALOG_TYPE: &str = "iceberg-rest";
|
||||
pub(crate) const TABLE_BUCKET_CONFIG_VERSION: u16 = 1;
|
||||
@@ -69,7 +69,7 @@ pub(crate) const TABLE_MAINTENANCE_CONFIG_VERSION: u16 = 1;
|
||||
pub(crate) const TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION: u16 = 1;
|
||||
pub(crate) const TABLE_CATALOG_BACKING_MANIFEST_VERSION: u16 = 1;
|
||||
pub(crate) const TABLE_METADATA_FILE_NAME_MAX_LEN: usize = 128;
|
||||
pub const TABLE_RESERVED_PREFIX: &str = ecstore_metadata::BUCKET_TABLE_RESERVED_PREFIX;
|
||||
pub const TABLE_RESERVED_PREFIX: &str = BUCKET_TABLE_RESERVED_PREFIX;
|
||||
const WAREHOUSE_ROOT: &str = "warehouses";
|
||||
const NAMESPACE_ROOT: &str = "namespaces";
|
||||
const TABLE_ROOT: &str = "tables";
|
||||
@@ -85,8 +85,8 @@ const TABLE_BUCKET_ENTRY_FILE: &str = "table-bucket.json";
|
||||
const NAMESPACE_ENTRY_FILE: &str = "namespace-entry.json";
|
||||
const TABLE_ENTRY_FILE: &str = "table-entry.json";
|
||||
const VIEW_ENTRY_FILE: &str = "view-entry.json";
|
||||
const INTERNAL_CATALOG_ROOT: &str = ecstore_metadata::BUCKET_TABLE_CATALOG_META_PREFIX;
|
||||
const TABLE_BUCKET_ROOT: &str = ecstore_metadata::BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX;
|
||||
const INTERNAL_CATALOG_ROOT: &str = BUCKET_TABLE_CATALOG_META_PREFIX;
|
||||
const TABLE_BUCKET_ROOT: &str = BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX;
|
||||
const COMMIT_LOG_ROOT: &str = "commits";
|
||||
const COMMIT_IDEMPOTENCY_ROOT: &str = "commit-idempotency";
|
||||
const EXTERNAL_CATALOG_ROOT: &str = "external-catalog";
|
||||
@@ -115,10 +115,8 @@ const ICEBERG_REF_MAX_REF_AGE_MS_FIELD: &str = "max-ref-age-ms";
|
||||
|
||||
type CatalogListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type CatalogListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type EcstoreError = ecstore_error::Error;
|
||||
type CatalogObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, EcstoreError>;
|
||||
type CatalogWalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
type StorageError = ecstore_error::StorageError;
|
||||
|
||||
pub(crate) trait TableCatalogStorage:
|
||||
StorageObjectIO<
|
||||
@@ -1463,7 +1461,7 @@ impl TableCatalogObjectPaths {
|
||||
"{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_CONFIG_FILE}",
|
||||
self.table_entries_prefix(table_bucket, namespace),
|
||||
table.as_str(),
|
||||
ecstore_metadata::table_catalog_path_hash(table_id)
|
||||
table_catalog_path_hash(table_id)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1479,8 +1477,8 @@ impl TableCatalogObjectPaths {
|
||||
"{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_JOB_ROOT}/{}.json",
|
||||
self.table_entries_prefix(table_bucket, namespace),
|
||||
table.as_str(),
|
||||
ecstore_metadata::table_catalog_path_hash(table_id),
|
||||
ecstore_metadata::table_catalog_path_hash(job_id)
|
||||
table_catalog_path_hash(table_id),
|
||||
table_catalog_path_hash(job_id)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1495,7 +1493,7 @@ impl TableCatalogObjectPaths {
|
||||
"{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_LATEST_JOB_FILE}",
|
||||
self.table_entries_prefix(table_bucket, namespace),
|
||||
table.as_str(),
|
||||
ecstore_metadata::table_catalog_path_hash(table_id)
|
||||
table_catalog_path_hash(table_id)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1510,7 +1508,7 @@ impl TableCatalogObjectPaths {
|
||||
"{}{}/{MAINTENANCE_ROOT}/{}/{MAINTENANCE_CURRENT_JOB_FILE}",
|
||||
self.table_entries_prefix(table_bucket, namespace),
|
||||
table.as_str(),
|
||||
ecstore_metadata::table_catalog_path_hash(table_id)
|
||||
table_catalog_path_hash(table_id)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1519,8 +1517,8 @@ impl TableCatalogObjectPaths {
|
||||
"{}{}/{}/{}.json",
|
||||
self.table_bucket_root_prefix(table_bucket),
|
||||
COMMIT_LOG_ROOT,
|
||||
ecstore_metadata::table_catalog_path_hash(table_id),
|
||||
ecstore_metadata::table_catalog_path_hash(commit_id)
|
||||
table_catalog_path_hash(table_id),
|
||||
table_catalog_path_hash(commit_id)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1529,7 +1527,7 @@ impl TableCatalogObjectPaths {
|
||||
"{}{}/{}/",
|
||||
self.table_bucket_root_prefix(table_bucket),
|
||||
COMMIT_LOG_ROOT,
|
||||
ecstore_metadata::table_catalog_path_hash(table_id)
|
||||
table_catalog_path_hash(table_id)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1538,8 +1536,8 @@ impl TableCatalogObjectPaths {
|
||||
"{}{}/{}/{}.json",
|
||||
self.table_bucket_root_prefix(table_bucket),
|
||||
COMMIT_IDEMPOTENCY_ROOT,
|
||||
ecstore_metadata::table_catalog_path_hash(table_id),
|
||||
ecstore_metadata::table_catalog_path_hash(idempotency_key)
|
||||
table_catalog_path_hash(table_id),
|
||||
table_catalog_path_hash(idempotency_key)
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1548,17 +1546,12 @@ impl TableCatalogObjectPaths {
|
||||
"{}{}/{}/",
|
||||
self.table_bucket_root_prefix(table_bucket),
|
||||
COMMIT_IDEMPOTENCY_ROOT,
|
||||
ecstore_metadata::table_catalog_path_hash(table_id)
|
||||
table_catalog_path_hash(table_id)
|
||||
)
|
||||
}
|
||||
|
||||
fn table_bucket_root_prefix(&self, table_bucket: &str) -> String {
|
||||
format!(
|
||||
"{}/{}/{}/",
|
||||
self.catalog_root,
|
||||
TABLE_BUCKET_ROOT,
|
||||
ecstore_metadata::table_catalog_path_hash(table_bucket)
|
||||
)
|
||||
format!("{}/{}/{}/", self.catalog_root, TABLE_BUCKET_ROOT, table_catalog_path_hash(table_bucket))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4038,7 +4031,7 @@ where
|
||||
.await
|
||||
.map_err(|err| storage_error_to_catalog("create catalog table lock", err))?;
|
||||
let guard = lock
|
||||
.get_write_lock(ecstore_set_disk::get_lock_acquire_timeout())
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| TableCatalogStoreError::Internal(format!("failed to acquire catalog table lock: {err}")))?;
|
||||
Ok(Box::new(guard))
|
||||
@@ -7009,7 +7002,7 @@ pub fn validate_object_mutation(table_bucket_enabled: bool, object_key: &str) ->
|
||||
}
|
||||
|
||||
pub(crate) async fn validate_bucket_object_mutation(bucket: &str, object_key: &str) -> Result<(), TableObjectMutationError> {
|
||||
let table_bucket_enabled = ecstore_metadata_sys::get(bucket)
|
||||
let table_bucket_enabled = get_bucket_metadata(bucket)
|
||||
.await
|
||||
.is_ok_and(|metadata| metadata.table_bucket_enabled());
|
||||
|
||||
@@ -7417,7 +7410,7 @@ mod tests {
|
||||
let bucket = "analytics";
|
||||
let namespace = Namespace::parse("analytics.daily_events").unwrap();
|
||||
let table = IdentifierSegment::parse("events").unwrap();
|
||||
let bucket_root = format!("s3tables/catalog/table-buckets/{}/", ecstore_metadata::table_catalog_path_hash(bucket));
|
||||
let bucket_root = format!("s3tables/catalog/table-buckets/{}/", table_catalog_path_hash(bucket));
|
||||
|
||||
assert_eq!(paths.table_bucket_entry_path(bucket), format!("{bucket_root}table-bucket.json"));
|
||||
assert_eq!(
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::ecstore_bucket::{metadata_sys, replication};
|
||||
use crate::storage::{bucket_metadata_runtime_initialized, get_global_replication_pool, replication_queue_current_count};
|
||||
use rustfs_concurrency::{
|
||||
AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider,
|
||||
WorkloadClass,
|
||||
@@ -75,7 +75,7 @@ pub fn foreground_read_workload_admission_snapshot() -> WorkloadAdmissionSnapsho
|
||||
}
|
||||
|
||||
pub fn metadata_workload_admission_snapshot() -> WorkloadAdmissionSnapshot {
|
||||
metadata_workload_admission_snapshot_from_initialized(metadata_sys::get_global_bucket_metadata_sys().is_some())
|
||||
metadata_workload_admission_snapshot_from_initialized(bucket_metadata_runtime_initialized())
|
||||
}
|
||||
|
||||
fn metadata_workload_admission_snapshot_from_initialized(runtime_initialized: bool) -> WorkloadAdmissionSnapshot {
|
||||
@@ -151,7 +151,7 @@ fn repair_workload_admission_snapshot_from_counts(
|
||||
}
|
||||
|
||||
pub fn replication_workload_admission_snapshot() -> WorkloadAdmissionSnapshot {
|
||||
let Some(pool) = replication::get_global_replication_pool() else {
|
||||
let Some(pool) = get_global_replication_pool() else {
|
||||
return replication_workload_admission_snapshot_from_counts(false, None, None);
|
||||
};
|
||||
|
||||
@@ -198,16 +198,6 @@ fn i32_to_usize_saturated(value: i32) -> usize {
|
||||
usize::try_from(value.max(0)).unwrap_or(usize::MAX)
|
||||
}
|
||||
|
||||
fn replication_queue_current_count() -> Option<i64> {
|
||||
replication::GLOBAL_REPLICATION_STATS.get().and_then(|stats| {
|
||||
stats
|
||||
.q_cache
|
||||
.try_lock()
|
||||
.ok()
|
||||
.map(|cache| cache.sr_queue_stats.curr.get_current_count())
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -101,6 +101,19 @@ ALL_ECSTORE_API_GROUPED_FACADE_IMPORT_HITS_FILE="${TMP_DIR}/all_ecstore_api_grou
|
||||
ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE="${TMP_DIR}/all_ecstore_api_raw_subpath_hits.txt"
|
||||
EXTERNAL_PRODUCTION_ECSTORE_IMPORT_HITS_FILE="${TMP_DIR}/external_production_ecstore_import_hits.txt"
|
||||
COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE="${TMP_DIR}/completed_external_owner_module_alias_hits.txt"
|
||||
COMPLETED_OWNER_BARE_FACADE_IMPORT_HITS_FILE="${TMP_DIR}/completed_owner_bare_facade_import_hits.txt"
|
||||
COMPLETED_OWNER_SCATTERED_RAW_FACADE_PATH_HITS_FILE="${TMP_DIR}/completed_owner_scattered_raw_facade_path_hits.txt"
|
||||
RUSTFS_STARTUP_OWNER_MODULE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_startup_owner_module_consumer_hits.txt"
|
||||
RUSTFS_RUNTIME_OWNER_MODULE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_runtime_owner_module_consumer_hits.txt"
|
||||
RUSTFS_ROOT_SERVER_OWNER_MODULE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_root_server_owner_module_consumer_hits.txt"
|
||||
RUSTFS_TABLE_S3_OWNER_MODULE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_table_s3_owner_module_consumer_hits.txt"
|
||||
RUSTFS_APP_SHARED_OWNER_MODULE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_app_shared_owner_module_consumer_hits.txt"
|
||||
RUSTFS_APP_BUCKET_OWNER_SOURCE_HITS_FILE="${TMP_DIR}/rustfs_app_bucket_owner_source_hits.txt"
|
||||
RUSTFS_APP_ECSTORE_SOURCE_HITS_FILE="${TMP_DIR}/rustfs_app_ecstore_source_hits.txt"
|
||||
RUSTFS_ADMIN_ECSTORE_SOURCE_HITS_FILE="${TMP_DIR}/rustfs_admin_ecstore_source_hits.txt"
|
||||
EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/external_runtime_ecstore_compat_bypass_hits.txt"
|
||||
EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/external_test_ecstore_compat_bypass_hits.txt"
|
||||
FUZZ_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/fuzz_ecstore_compat_bypass_hits.txt"
|
||||
ALL_STORAGE_COMPAT_SELF_FACADE_PATH_HITS_FILE="${TMP_DIR}/all_storage_compat_self_facade_path_hits.txt"
|
||||
RUSTFS_LOCAL_COMPAT_OWNER_SELF_PATH_HITS_FILE="${TMP_DIR}/rustfs_local_compat_owner_self_path_hits.txt"
|
||||
RUSTFS_ROOT_COMPAT_RELATIVE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_root_compat_relative_consumer_hits.txt"
|
||||
@@ -718,8 +731,12 @@ fi
|
||||
--glob '!crates/ecstore/**' \
|
||||
--glob '!**/storage_compat.rs' \
|
||||
--glob '!**/*storage_compat.rs' \
|
||||
--glob '!**/ecstore_compat.rs' \
|
||||
--glob '!**/ecstore_test_compat.rs' \
|
||||
--glob '!**/ecstore_test_compat/**' \
|
||||
--glob '!**/ecstore_fuzz_compat.rs' \
|
||||
--glob '!target/**' \
|
||||
| rg -v '^(rustfs/src/(admin/mod|app/mod|storage/mod)\.rs|crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)\.rs|crates/iam/src/lib\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs|fuzz/fuzz_targets/(bucket_validation|path_containment)\.rs):' || true
|
||||
| rg -v '^(rustfs/src/(admin/mod|app/mod)\.rs|crates/e2e_test/src/(replication_extension_test|reliant/(grpc_lock_client|node_interact_test))\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/(endpoint_index_test|heal_bug_fixes_test|heal_integration_test)\.rs|crates/iam/src/lib\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs|fuzz/fuzz_targets/(bucket_validation|path_containment)\.rs):' || true
|
||||
) |
|
||||
cat >"$DIRECT_ECSTORE_IMPORT_HITS_FILE"
|
||||
|
||||
@@ -1028,6 +1045,9 @@ fi
|
||||
rg -n --with-filename 'rustfs_ecstore::api::' rustfs/src crates fuzz \
|
||||
--glob '*storage_compat.rs' \
|
||||
--glob '*_compat.rs' \
|
||||
--glob '!**/ecstore_compat.rs' \
|
||||
--glob '!**/ecstore_test_compat.rs' \
|
||||
--glob '!**/ecstore_test_compat/**' \
|
||||
| rg -v '^[^:]+:[0-9]+:use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' || true
|
||||
) >"$ALL_STORAGE_COMPAT_RAW_FACADE_PATH_HITS_FILE"
|
||||
|
||||
@@ -1061,8 +1081,11 @@ fi
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'rustfs_ecstore::api::[a-z_]+::' rustfs/src crates fuzz \
|
||||
--glob '*.rs' \
|
||||
--glob '!crates/ecstore/**' |
|
||||
rg -v '^(fuzz/fuzz_targets/bucket_validation\.rs|fuzz/fuzz_targets/path_containment\.rs|crates/e2e_test/src/reliant/grpc_lock_client\.rs|crates/e2e_test/src/reliant/node_interact_test\.rs|crates/e2e_test/src/replication_extension_test\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/endpoint_index_test\.rs|crates/heal/tests/heal_bug_fixes_test\.rs|crates/heal/tests/heal_integration_test\.rs|crates/iam/src/lib\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs):' || true
|
||||
--glob '!crates/ecstore/**' \
|
||||
--glob '!**/ecstore_compat.rs' \
|
||||
--glob '!**/ecstore_test_compat.rs' \
|
||||
--glob '!**/ecstore_test_compat/**' |
|
||||
rg -v '^(fuzz/fuzz_targets/bucket_validation\.rs|fuzz/fuzz_targets/path_containment\.rs|crates/e2e_test/src/reliant/grpc_lock_client\.rs|crates/e2e_test/src/reliant/node_interact_test\.rs|crates/e2e_test/src/replication_extension_test\.rs|crates/heal/src/heal/mod\.rs|crates/heal/tests/endpoint_index_test\.rs|crates/heal/tests/heal_bug_fixes_test\.rs|crates/heal/tests/heal_integration_test\.rs|crates/iam/src/lib\.rs|crates/notify/src/lib\.rs|crates/obs/src/metrics/mod\.rs|crates/protocols/src/swift/mod\.rs|crates/s3select-api/src/lib\.rs|crates/scanner/src/lib\.rs|crates/scanner/tests/lifecycle_integration_test\.rs|rustfs/src/admin/mod\.rs|rustfs/src/app/mod\.rs):' || true
|
||||
) >"$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE"
|
||||
|
||||
if [[ -s "$ALL_ECSTORE_API_RAW_SUBPATH_HITS_FILE" ]]; then
|
||||
@@ -1086,6 +1109,9 @@ fi
|
||||
crates/e2e_test/src/reliant/grpc_lock_client.rs \
|
||||
crates/e2e_test/src/reliant/node_interact_test.rs \
|
||||
crates/e2e_test/src/replication_extension_test.rs \
|
||||
rustfs/src/admin/mod.rs \
|
||||
rustfs/src/app/mod.rs \
|
||||
rustfs/src/storage/mod.rs \
|
||||
fuzz/fuzz_targets/bucket_validation.rs \
|
||||
fuzz/fuzz_targets/path_containment.rs || true
|
||||
) >"$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE"
|
||||
@@ -1094,6 +1120,205 @@ if [[ -s "$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE" ]]; then
|
||||
report_failure "completed external owner and test/fuzz boundaries must expose explicit ECStore symbols instead of ecstore_* module aliases: $(paste -sd '; ' "$COMPLETED_EXTERNAL_OWNER_MODULE_ALIAS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename '^(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+\s*;|^(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+::\*\s*;' \
|
||||
crates/heal/src/heal/mod.rs \
|
||||
crates/heal/tests/endpoint_index_test.rs \
|
||||
crates/heal/tests/heal_bug_fixes_test.rs \
|
||||
crates/heal/tests/heal_integration_test.rs \
|
||||
crates/iam/src/lib.rs \
|
||||
crates/notify/src/lib.rs \
|
||||
crates/obs/src/metrics/mod.rs \
|
||||
crates/protocols/src/swift/mod.rs \
|
||||
crates/s3select-api/src/lib.rs \
|
||||
crates/scanner/src/lib.rs \
|
||||
crates/scanner/tests/lifecycle_integration_test.rs \
|
||||
crates/e2e_test/src/reliant/grpc_lock_client.rs \
|
||||
crates/e2e_test/src/reliant/node_interact_test.rs \
|
||||
crates/e2e_test/src/replication_extension_test.rs \
|
||||
rustfs/src/admin/mod.rs \
|
||||
rustfs/src/app/mod.rs \
|
||||
rustfs/src/storage/mod.rs \
|
||||
fuzz/fuzz_targets/bucket_validation.rs \
|
||||
fuzz/fuzz_targets/path_containment.rs || true
|
||||
) >"$COMPLETED_OWNER_BARE_FACADE_IMPORT_HITS_FILE"
|
||||
|
||||
if [[ -s "$COMPLETED_OWNER_BARE_FACADE_IMPORT_HITS_FILE" ]]; then
|
||||
report_failure "completed owner and test/fuzz boundaries must not import bare or glob ECStore facade modules: $(paste -sd '; ' "$COMPLETED_OWNER_BARE_FACADE_IMPORT_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'rustfs_ecstore::api::[a-z_]+::' \
|
||||
crates/heal/src/heal/mod.rs \
|
||||
crates/heal/tests/endpoint_index_test.rs \
|
||||
crates/heal/tests/heal_bug_fixes_test.rs \
|
||||
crates/heal/tests/heal_integration_test.rs \
|
||||
crates/iam/src/lib.rs \
|
||||
crates/notify/src/lib.rs \
|
||||
crates/obs/src/metrics/mod.rs \
|
||||
crates/protocols/src/swift/mod.rs \
|
||||
crates/s3select-api/src/lib.rs \
|
||||
crates/scanner/src/lib.rs \
|
||||
crates/scanner/tests/lifecycle_integration_test.rs \
|
||||
crates/e2e_test/src/reliant/grpc_lock_client.rs \
|
||||
crates/e2e_test/src/reliant/node_interact_test.rs \
|
||||
crates/e2e_test/src/replication_extension_test.rs \
|
||||
rustfs/src/admin/mod.rs \
|
||||
rustfs/src/app/mod.rs \
|
||||
rustfs/src/storage/mod.rs \
|
||||
fuzz/fuzz_targets/bucket_validation.rs \
|
||||
fuzz/fuzz_targets/path_containment.rs |
|
||||
rg -v '^[^:]+:[0-9]+:\s*(?:pub\(crate\)\s+)?use\s+rustfs_ecstore::api::[a-z_]+::' || true
|
||||
) >"$COMPLETED_OWNER_SCATTERED_RAW_FACADE_PATH_HITS_FILE"
|
||||
|
||||
if [[ -s "$COMPLETED_OWNER_SCATTERED_RAW_FACADE_PATH_HITS_FILE" ]]; then
|
||||
report_failure "completed owner and test/fuzz boundaries must keep raw ECStore facade subpaths in explicit import declarations: $(paste -sd '; ' "$COMPLETED_OWNER_SCATTERED_RAW_FACADE_PATH_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::storage::.*ecstore_|^\s*ecstore_[a-z_]+(?:::|,|\})' \
|
||||
rustfs/src/startup_notification.rs \
|
||||
rustfs/src/startup_fs_guard.rs \
|
||||
rustfs/src/startup_services.rs \
|
||||
rustfs/src/startup_server.rs \
|
||||
rustfs/src/startup_storage.rs || true
|
||||
) >"$RUSTFS_STARTUP_OWNER_MODULE_CONSUMER_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_STARTUP_OWNER_MODULE_CONSUMER_HITS_FILE" ]]; then
|
||||
report_failure "RustFS startup runtime consumers must use storage owner symbols instead of ecstore_* modules: $(paste -sd '; ' "$RUSTFS_STARTUP_OWNER_MODULE_CONSUMER_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::storage::.*ecstore_|^\s*ecstore_[a-z_]+(?:::|,|\})' \
|
||||
rustfs/src/server/http.rs \
|
||||
rustfs/src/capacity/service.rs \
|
||||
rustfs/src/workload_admission.rs || true
|
||||
) >"$RUSTFS_RUNTIME_OWNER_MODULE_CONSUMER_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_RUNTIME_OWNER_MODULE_CONSUMER_HITS_FILE" ]]; then
|
||||
report_failure "RustFS runtime server/capacity/workload consumers must use storage owner symbols instead of ecstore_* modules: $(paste -sd '; ' "$RUSTFS_RUNTIME_OWNER_MODULE_CONSUMER_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::storage::.*ecstore_|^\s*ecstore_[a-z_]+(?:::|,|\})' \
|
||||
rustfs/src/init.rs \
|
||||
rustfs/src/runtime_capabilities.rs \
|
||||
rustfs/src/server/readiness.rs \
|
||||
rustfs/src/server/event.rs \
|
||||
rustfs/src/server/module_switch.rs \
|
||||
rustfs/src/error.rs || true
|
||||
) >"$RUSTFS_ROOT_SERVER_OWNER_MODULE_CONSUMER_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_ROOT_SERVER_OWNER_MODULE_CONSUMER_HITS_FILE" ]]; then
|
||||
report_failure "RustFS root/server runtime consumers must use storage owner symbols instead of ecstore_* modules: $(paste -sd '; ' "$RUSTFS_ROOT_SERVER_OWNER_MODULE_CONSUMER_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::storage::.*ecstore_|^\s*ecstore_[a-z_]+(?:::|,|\})' \
|
||||
rustfs/src/startup_bucket_metadata.rs \
|
||||
rustfs/src/startup_shutdown.rs \
|
||||
rustfs/src/table_catalog.rs \
|
||||
rustfs/src/storage/s3_api/bucket.rs \
|
||||
rustfs/src/storage/s3_api/multipart.rs \
|
||||
rustfs/src/config/config_test.rs || true
|
||||
) >"$RUSTFS_TABLE_S3_OWNER_MODULE_CONSUMER_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_TABLE_S3_OWNER_MODULE_CONSUMER_HITS_FILE" ]]; then
|
||||
report_failure "RustFS table/S3/startup consumers must use storage owner symbols instead of ecstore_* modules: $(paste -sd '; ' "$RUSTFS_TABLE_S3_OWNER_MODULE_CONSUMER_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'ecstore_(?:disk::(?:error::DiskError|endpoint::Endpoint|error_reduce::is_all_buckets_not_found)|error::(?:StorageError|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)|global::(?:get_global_endpoints_opt|get_global_region|get_global_tier_config_mgr|new_object_layer_fn|set_object_store_resolver)|layout::(?:EndpointServerPools|Endpoints|PoolEndpoints)|notification::(?:NotificationSys|get_global_notification_sys)|rio::(?:DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|compression_metadata_value|wrap_reader)|set_disk::(?:get_lock_acquire_timeout|is_valid_storage_class)|storage::(?:ECStore|init_local_disks))' \
|
||||
rustfs/src/app/mod.rs || true
|
||||
) >"$RUSTFS_APP_SHARED_OWNER_MODULE_CONSUMER_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_APP_SHARED_OWNER_MODULE_CONSUMER_HITS_FILE" ]]; then
|
||||
report_failure "RustFS app shared runtime facade must use storage owner symbols instead of duplicate ecstore_* calls: $(paste -sd '; ' "$RUSTFS_APP_SHARED_OWNER_MODULE_CONSUMER_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'rustfs_ecstore::api::(?:bucket|client|config)::' \
|
||||
rustfs/src/app/mod.rs || true
|
||||
) >"$RUSTFS_APP_BUCKET_OWNER_SOURCE_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_APP_BUCKET_OWNER_SOURCE_HITS_FILE" ]]; then
|
||||
report_failure "RustFS app bucket facade must source completed bucket/client/config symbols through storage owner re-exports: $(paste -sd '; ' "$RUSTFS_APP_BUCKET_OWNER_SOURCE_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'rustfs_ecstore::api::' \
|
||||
rustfs/src/app/mod.rs || true
|
||||
) >"$RUSTFS_APP_ECSTORE_SOURCE_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_APP_ECSTORE_SOURCE_HITS_FILE" ]]; then
|
||||
report_failure "RustFS app facade must source ECStore API symbols through storage owner re-exports: $(paste -sd '; ' "$RUSTFS_APP_ECSTORE_SOURCE_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'rustfs_ecstore::api::' \
|
||||
rustfs/src/admin/mod.rs || true
|
||||
) >"$RUSTFS_ADMIN_ECSTORE_SOURCE_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_ADMIN_ECSTORE_SOURCE_HITS_FILE" ]]; then
|
||||
report_failure "RustFS admin facade must source ECStore API symbols through storage owner re-exports: $(paste -sd '; ' "$RUSTFS_ADMIN_ECSTORE_SOURCE_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'rustfs_ecstore::api::' \
|
||||
crates/heal/src/heal \
|
||||
crates/iam/src \
|
||||
crates/notify/src \
|
||||
crates/obs/src/metrics \
|
||||
crates/protocols/src/swift \
|
||||
crates/s3select-api/src \
|
||||
crates/scanner/src \
|
||||
--glob '*.rs' \
|
||||
--glob '!**/ecstore_compat.rs' |
|
||||
rg -v '^(crates/notify/src/lib.rs|crates/obs/src/metrics/mod.rs|crates/protocols/src/swift/mod.rs|crates/s3select-api/src/lib.rs):' || true
|
||||
) >"$EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE"
|
||||
|
||||
if [[ -s "$EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE" ]]; then
|
||||
report_failure "external runtime crates must source ECStore API symbols through their owner root or ecstore_compat boundary: $(paste -sd '; ' "$EXTERNAL_RUNTIME_ECSTORE_COMPAT_BYPASS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'rustfs_ecstore::api::' \
|
||||
crates/heal/tests \
|
||||
crates/scanner/tests \
|
||||
crates/e2e_test/src \
|
||||
--glob '*.rs' \
|
||||
--glob '!**/ecstore_test_compat.rs' \
|
||||
--glob '!**/ecstore_test_compat/**' || true
|
||||
) >"$EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE"
|
||||
|
||||
if [[ -s "$EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE" ]]; then
|
||||
report_failure "external test crates must source ECStore API symbols through their ecstore_test_compat boundary: $(paste -sd '; ' "$EXTERNAL_TEST_ECSTORE_COMPAT_BYPASS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'rustfs_ecstore::api::' \
|
||||
fuzz/fuzz_targets \
|
||||
--glob '*.rs' \
|
||||
--glob '!**/ecstore_fuzz_compat.rs' || true
|
||||
) >"$FUZZ_ECSTORE_COMPAT_BYPASS_HITS_FILE"
|
||||
|
||||
if [[ -s "$FUZZ_ECSTORE_COMPAT_BYPASS_HITS_FILE" ]]; then
|
||||
report_failure "fuzz targets must source ECStore API symbols through ecstore_fuzz_compat: $(paste -sd '; ' "$FUZZ_ECSTORE_COMPAT_BYPASS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename '^(?:pub\(crate\) )?use rustfs_ecstore::api::[a-z_]+ as ecstore_[a-z_]+;' \
|
||||
@@ -1383,9 +1608,13 @@ fi
|
||||
for file in \
|
||||
crates/e2e_test/src/storage_compat.rs \
|
||||
crates/iam/src/store/storage_compat.rs \
|
||||
crates/notify/src/ecstore_compat.rs \
|
||||
crates/notify/src/storage_compat.rs \
|
||||
crates/obs/src/metrics/ecstore_compat.rs \
|
||||
crates/obs/src/storage_compat.rs \
|
||||
crates/protocols/src/swift/ecstore_compat.rs \
|
||||
crates/protocols/src/swift/storage_compat.rs \
|
||||
crates/s3select-api/src/ecstore_compat.rs \
|
||||
crates/s3select-api/src/storage_compat.rs; do
|
||||
[[ -e "$file" ]] && printf '%s:1:standalone thin bridge file exists\n' "$file"
|
||||
done
|
||||
@@ -1396,6 +1625,12 @@ fi
|
||||
crates/protocols/src/swift \
|
||||
crates/s3select-api/src \
|
||||
-g '*.rs' || true
|
||||
rg -n --with-filename 'ecstore_compat' \
|
||||
crates/notify/src \
|
||||
crates/obs/src/metrics \
|
||||
crates/protocols/src/swift \
|
||||
crates/s3select-api/src \
|
||||
-g '*.rs' || true
|
||||
rg -n --with-filename '^\s*use\s+super::storage_compat|store::storage_compat|\bmod\s+storage_compat' \
|
||||
crates/iam/src/store.rs \
|
||||
crates/iam/src/store/object.rs || true
|
||||
@@ -1403,7 +1638,7 @@ fi
|
||||
) >"$STANDALONE_THIN_COMPAT_BRIDGE_HITS_FILE"
|
||||
|
||||
if [[ -s "$STANDALONE_THIN_COMPAT_BRIDGE_HITS_FILE" ]]; then
|
||||
report_failure "standalone e2e/IAM-store/notify consumers must import owner APIs directly instead of local storage compatibility bridges: $(paste -sd '; ' "$STANDALONE_THIN_COMPAT_BRIDGE_HITS_FILE")"
|
||||
report_failure "standalone e2e/IAM-store/notify/obs/swift/s3select consumers must import owner APIs directly instead of local thin compatibility bridges: $(paste -sd '; ' "$STANDALONE_THIN_COMPAT_BRIDGE_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
|
||||
Reference in New Issue
Block a user