mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
refactor: flatten test harness storage compat aliases (#3596)
* refactor: flatten test harness storage compat aliases * refactor: flatten rustfs storage compat aliases (#3597) * refactor: prune runtime storage compat surface (#3598) * refactor: flatten runtime secondary storage compat (#3599) * docs: add scheduler placement profiling baselines (#3600) * feat: add observability topology capability contracts (#3601)
This commit is contained in:
@@ -15,7 +15,7 @@
|
||||
// Used by test_distributed_lock_4_nodes_grpc in lock.rs
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::storage_compat::ecstore::rpc::node_service_time_out_client_no_auth;
|
||||
use crate::storage_compat::node_service_time_out_client_no_auth;
|
||||
use async_trait::async_trait;
|
||||
use rustfs_lock::{
|
||||
LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
@@ -41,10 +41,7 @@ impl GrpcLockClient {
|
||||
&self,
|
||||
) -> Result<
|
||||
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
|
||||
tonic::service::interceptor::InterceptedService<
|
||||
tonic::transport::Channel,
|
||||
crate::storage_compat::ecstore::rpc::TonicInterceptor,
|
||||
>,
|
||||
tonic::service::interceptor::InterceptedService<tonic::transport::Channel, crate::storage_compat::TonicInterceptor>,
|
||||
>,
|
||||
> {
|
||||
node_service_time_out_client_no_auth(&self.addr)
|
||||
|
||||
@@ -14,8 +14,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::workspace_root;
|
||||
use crate::storage_compat::ecstore::disk::{VolumeInfo, WalkDirOptions};
|
||||
use crate::storage_compat::ecstore::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
use crate::storage_compat::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
use crate::storage_compat::{VolumeInfo, WalkDirOptions};
|
||||
use futures::future::join_all;
|
||||
use rmp_serde::{Deserializer, Serializer};
|
||||
use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use crate::common::{
|
||||
RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
|
||||
};
|
||||
use crate::storage_compat::ecstore::bucket::bucket_target_sys::BucketTargetSys;
|
||||
use crate::storage_compat::BucketTargetSys;
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
@@ -12,22 +12,13 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod ecstore {
|
||||
#![allow(unused_imports)]
|
||||
#![allow(dead_code, unused_imports)]
|
||||
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) mod bucket_target_sys {
|
||||
pub(crate) use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
|
||||
}
|
||||
}
|
||||
pub(crate) type BucketTargetSys = rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
|
||||
pub(crate) type TonicInterceptor = rustfs_ecstore::rpc::TonicInterceptor;
|
||||
pub(crate) type VolumeInfo = rustfs_ecstore::disk::VolumeInfo;
|
||||
pub(crate) type WalkDirOptions = rustfs_ecstore::disk::WalkDirOptions;
|
||||
|
||||
pub(crate) mod disk {
|
||||
pub(crate) use rustfs_ecstore::disk::{VolumeInfo, WalkDirOptions};
|
||||
}
|
||||
|
||||
pub(crate) mod rpc {
|
||||
pub(crate) use rustfs_ecstore::rpc::{
|
||||
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
};
|
||||
}
|
||||
}
|
||||
pub(crate) use rustfs_ecstore::rpc::{
|
||||
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||
};
|
||||
|
||||
@@ -12,28 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod ecstore {
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_imports)]
|
||||
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) mod metadata_sys {
|
||||
pub(crate) use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod disk {
|
||||
pub(crate) use rustfs_ecstore::disk::DiskStore;
|
||||
|
||||
pub(crate) mod endpoint {
|
||||
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod endpoints {
|
||||
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
}
|
||||
|
||||
pub(crate) mod store {
|
||||
pub(crate) use rustfs_ecstore::store::{ECStore, init_local_disks};
|
||||
}
|
||||
}
|
||||
pub(crate) use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
|
||||
pub(crate) use rustfs_ecstore::disk::{DiskStore, endpoint::Endpoint};
|
||||
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
pub(crate) use rustfs_ecstore::store::ECStore;
|
||||
pub(crate) use rustfs_ecstore::store::init_local_disks;
|
||||
|
||||
@@ -16,8 +16,7 @@
|
||||
|
||||
mod common;
|
||||
|
||||
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
|
||||
use crate::common::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::common::storage_compat::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks};
|
||||
use std::net::SocketAddr;
|
||||
use tempfile::TempDir;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -73,12 +72,10 @@ async fn test_endpoint_index_settings() -> anyhow::Result<()> {
|
||||
}
|
||||
|
||||
// test ECStore initialization
|
||||
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone()).await?;
|
||||
init_local_disks(endpoint_pools.clone()).await?;
|
||||
|
||||
let server_addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||
let ecstore =
|
||||
crate::common::storage_compat::ecstore::store::ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
|
||||
.await?;
|
||||
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()).await?;
|
||||
|
||||
println!("ECStore initialized successfully with {} pools", ecstore.pools.len());
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ use rustfs_heal::heal::{
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_to_heal_request_no_panic() {
|
||||
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
|
||||
use crate::common::storage_compat::Endpoint;
|
||||
|
||||
// Test that invalid pool/set indices don't cause panic
|
||||
// Create endpoint using try_from or similar method
|
||||
@@ -47,7 +47,7 @@ fn test_heal_event_to_heal_request_no_panic() {
|
||||
|
||||
#[test]
|
||||
fn test_heal_event_to_heal_request_valid_indices() {
|
||||
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
|
||||
use crate::common::storage_compat::Endpoint;
|
||||
|
||||
// Test that valid indices work correctly
|
||||
let endpoint_result = Endpoint::try_from("http://localhost:9000");
|
||||
@@ -192,14 +192,11 @@ fn test_heal_task_status_atomic_update() {
|
||||
}
|
||||
async fn get_disk_status(
|
||||
&self,
|
||||
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
|
||||
_endpoint: &crate::common::storage_compat::Endpoint,
|
||||
) -> rustfs_heal::Result<rustfs_heal::heal::storage::DiskStatus> {
|
||||
Ok(rustfs_heal::heal::storage::DiskStatus::Ok)
|
||||
}
|
||||
async fn format_disk(
|
||||
&self,
|
||||
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
|
||||
) -> rustfs_heal::Result<()> {
|
||||
async fn format_disk(&self, _endpoint: &crate::common::storage_compat::Endpoint) -> rustfs_heal::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_storage_api::BucketInfo>> {
|
||||
@@ -253,10 +250,7 @@ fn test_heal_task_status_atomic_update() {
|
||||
) -> rustfs_heal::Result<(Vec<String>, Option<String>, bool)> {
|
||||
Ok((vec![], None, false))
|
||||
}
|
||||
async fn get_disk_for_resume(
|
||||
&self,
|
||||
_set_disk_id: &str,
|
||||
) -> rustfs_heal::Result<crate::common::storage_compat::ecstore::disk::DiskStore> {
|
||||
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<crate::common::storage_compat::DiskStore> {
|
||||
Err(rustfs_heal::Error::other("Not implemented in mock"))
|
||||
}
|
||||
}
|
||||
@@ -326,17 +320,11 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn get_disk_status(
|
||||
&self,
|
||||
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
|
||||
) -> rustfs_heal::Result<DiskStatus> {
|
||||
async fn get_disk_status(&self, _endpoint: &crate::common::storage_compat::Endpoint) -> rustfs_heal::Result<DiskStatus> {
|
||||
Ok(DiskStatus::Ok)
|
||||
}
|
||||
|
||||
async fn format_disk(
|
||||
&self,
|
||||
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
|
||||
) -> rustfs_heal::Result<()> {
|
||||
async fn format_disk(&self, _endpoint: &crate::common::storage_compat::Endpoint) -> rustfs_heal::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -406,10 +394,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
|
||||
Ok((Vec::new(), None, false))
|
||||
}
|
||||
|
||||
async fn get_disk_for_resume(
|
||||
&self,
|
||||
_set_disk_id: &str,
|
||||
) -> rustfs_heal::Result<crate::common::storage_compat::ecstore::disk::DiskStore> {
|
||||
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<crate::common::storage_compat::DiskStore> {
|
||||
Err(rustfs_heal::Error::other("not implemented"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,10 +14,8 @@
|
||||
|
||||
mod common;
|
||||
|
||||
use crate::common::storage_compat::ecstore::{
|
||||
disk::endpoint::Endpoint,
|
||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
store::ECStore,
|
||||
use crate::common::storage_compat::{
|
||||
ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
|
||||
};
|
||||
use http::HeaderMap;
|
||||
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
|
||||
@@ -124,9 +122,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
|
||||
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
||||
|
||||
// format disks (only first time)
|
||||
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
|
||||
// create ECStore with dynamic port 0 (let OS assign) or fixed 9001 if free
|
||||
let port = 9001; // for simplicity
|
||||
@@ -144,7 +140,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
|
||||
.await
|
||||
.unwrap();
|
||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
|
||||
// Create heal storage layer
|
||||
let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
|
||||
|
||||
@@ -12,70 +12,27 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod ecstore {
|
||||
#![allow(unused_imports)]
|
||||
#![allow(dead_code, unused_imports)]
|
||||
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) mod lifecycle {
|
||||
pub(crate) use rustfs_ecstore::bucket::lifecycle::lifecycle::TransitionOptions;
|
||||
pub(crate) const BUCKET_LIFECYCLE_CONFIG: &str = rustfs_ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
||||
pub(crate) const STORAGE_FORMAT_FILE: &str = rustfs_ecstore::disk::STORAGE_FORMAT_FILE;
|
||||
|
||||
pub(crate) mod bucket_lifecycle_ops {
|
||||
pub(crate) use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
enqueue_transition_for_existing_objects, init_background_expiry,
|
||||
};
|
||||
}
|
||||
}
|
||||
pub(crate) type TransitionOptions = rustfs_ecstore::bucket::lifecycle::lifecycle::TransitionOptions;
|
||||
|
||||
pub(crate) mod metadata {
|
||||
pub(crate) use rustfs_ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
||||
}
|
||||
|
||||
pub(crate) mod metadata_sys {
|
||||
pub(crate) use rustfs_ecstore::bucket::metadata_sys::{get, init_bucket_metadata_sys, update};
|
||||
}
|
||||
|
||||
pub(crate) mod versioning_sys {
|
||||
pub(crate) use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod client {
|
||||
pub(crate) mod transition_api {
|
||||
pub(crate) use rustfs_ecstore::client::transition_api::{ReadCloser, ReaderImpl};
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod disk {
|
||||
pub(crate) use rustfs_ecstore::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, new_disk};
|
||||
|
||||
pub(crate) mod endpoint {
|
||||
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod endpoints {
|
||||
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
}
|
||||
|
||||
pub(crate) mod global {
|
||||
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
|
||||
}
|
||||
|
||||
pub(crate) mod pools {
|
||||
pub(crate) use rustfs_ecstore::pools::path2_bucket_object_with_base_path;
|
||||
}
|
||||
|
||||
pub(crate) mod store {
|
||||
pub(crate) use rustfs_ecstore::store::{ECStore, init_local_disks};
|
||||
}
|
||||
|
||||
pub(crate) mod tier {
|
||||
pub(crate) mod tier_config {
|
||||
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierMinIO, TierType};
|
||||
}
|
||||
|
||||
pub(crate) mod warm_backend {
|
||||
pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options};
|
||||
}
|
||||
}
|
||||
}
|
||||
pub(crate) use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
enqueue_transition_for_existing_objects, init_background_expiry,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::bucket::metadata_sys::{
|
||||
get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
|
||||
pub(crate) use rustfs_ecstore::client::transition_api::{ReadCloser, ReaderImpl};
|
||||
pub(crate) use rustfs_ecstore::disk::{DiskAPI, DiskOption, endpoint::Endpoint, new_disk};
|
||||
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
|
||||
pub(crate) use rustfs_ecstore::pools::path2_bucket_object_with_base_path;
|
||||
pub(crate) use rustfs_ecstore::store::ECStore;
|
||||
pub(crate) use rustfs_ecstore::store::init_local_disks;
|
||||
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierMinIO, TierType};
|
||||
pub(crate) use rustfs_ecstore::tier::warm_backend::WarmBackendGetOpts;
|
||||
pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, build_transition_put_options};
|
||||
|
||||
@@ -14,24 +14,12 @@
|
||||
|
||||
mod common;
|
||||
|
||||
use crate::common::storage_compat::ecstore::{
|
||||
bucket::metadata::BUCKET_LIFECYCLE_CONFIG,
|
||||
bucket::{
|
||||
lifecycle::{TransitionOptions, bucket_lifecycle_ops::enqueue_transition_for_existing_objects},
|
||||
metadata_sys,
|
||||
versioning_sys::BucketVersioningSys,
|
||||
},
|
||||
client::transition_api::{ReadCloser, ReaderImpl},
|
||||
disk::endpoint::Endpoint,
|
||||
disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, new_disk},
|
||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
global::GLOBAL_TierConfigMgr,
|
||||
pools::path2_bucket_object_with_base_path,
|
||||
store::ECStore,
|
||||
tier::{
|
||||
tier_config::{TierConfig, TierMinIO, TierType},
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
||||
},
|
||||
use crate::common::storage_compat::{
|
||||
BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DiskAPI, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
|
||||
GLOBAL_TierConfigMgr, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, TierConfig, TierMinIO, TierType,
|
||||
TransitionOptions, WarmBackend, WarmBackendGetOpts, build_transition_put_options, enqueue_transition_for_existing_objects,
|
||||
get_bucket_metadata, init_background_expiry, init_bucket_metadata_sys, init_local_disks, new_disk,
|
||||
path2_bucket_object_with_base_path, update_bucket_metadata,
|
||||
};
|
||||
use futures::FutureExt;
|
||||
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
|
||||
@@ -125,9 +113,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
||||
|
||||
// format disks (only first time)
|
||||
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
|
||||
// create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
|
||||
let port = 9002; // for simplicity
|
||||
@@ -145,11 +131,10 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
.await
|
||||
.unwrap();
|
||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
|
||||
// Initialize background expiry workers
|
||||
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
|
||||
.await;
|
||||
init_background_expiry(ecstore.clone()).await;
|
||||
|
||||
// Store in global once lock
|
||||
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
|
||||
@@ -197,9 +182,7 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
|
||||
};
|
||||
|
||||
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
||||
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||
|
||||
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
|
||||
@@ -214,11 +197,10 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
|
||||
.await
|
||||
.unwrap();
|
||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
|
||||
if init_expiry {
|
||||
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
|
||||
.await;
|
||||
init_background_expiry(ecstore.clone()).await;
|
||||
}
|
||||
|
||||
(disk_paths, ecstore)
|
||||
@@ -287,7 +269,7 @@ async fn set_bucket_lifecycle(bucket_name: &str) -> Result<(), Box<dyn std::erro
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#;
|
||||
|
||||
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
|
||||
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -311,7 +293,7 @@ async fn set_bucket_lifecycle_deletemarker(bucket_name: &str) -> Result<(), Box<
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#;
|
||||
|
||||
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
|
||||
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -334,7 +316,7 @@ async fn set_bucket_lifecycle_delmarker_expiration(bucket_name: &str, days: i64)
|
||||
</LifecycleConfiguration>"#
|
||||
);
|
||||
|
||||
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
|
||||
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -378,7 +360,7 @@ async fn set_bucket_lifecycle_transition_with_tier(
|
||||
</LifecycleConfiguration>"#
|
||||
);
|
||||
|
||||
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
|
||||
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -613,7 +595,7 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
|
||||
.await
|
||||
.expect("failed to stat object metadata")
|
||||
.file_type();
|
||||
let lifecycle = metadata_sys::get(bucket)
|
||||
let lifecycle = get_bucket_metadata(bucket)
|
||||
.await
|
||||
.expect("failed to load bucket metadata")
|
||||
.lifecycle_config
|
||||
@@ -878,7 +860,7 @@ mod serial_tests {
|
||||
println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
|
||||
|
||||
// Verify lifecycle configuration was set
|
||||
match crate::common::storage_compat::ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
|
||||
match get_bucket_metadata(bucket_name.as_str()).await {
|
||||
Ok(bucket_meta) => {
|
||||
assert!(bucket_meta.lifecycle_config.is_some());
|
||||
println!("✅ Bucket metadata retrieved successfully");
|
||||
@@ -1284,8 +1266,7 @@ mod serial_tests {
|
||||
"stale transitioned remote object should still exist before scanner fallback runs"
|
||||
);
|
||||
|
||||
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
|
||||
.await;
|
||||
init_background_expiry(ecstore.clone()).await;
|
||||
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||
|
||||
assert!(
|
||||
@@ -1346,8 +1327,7 @@ mod serial_tests {
|
||||
"stale transitioned remote object should still exist before scanner cleanup runs"
|
||||
);
|
||||
|
||||
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
|
||||
.await;
|
||||
init_background_expiry(ecstore.clone()).await;
|
||||
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||
|
||||
assert!(
|
||||
@@ -1437,7 +1417,7 @@ mod serial_tests {
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#
|
||||
);
|
||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
|
||||
@@ -1522,7 +1502,7 @@ mod serial_tests {
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#
|
||||
);
|
||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
|
||||
@@ -1714,8 +1694,7 @@ mod serial_tests {
|
||||
|
||||
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
|
||||
|
||||
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
|
||||
.await;
|
||||
init_background_expiry(ecstore.clone()).await;
|
||||
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||
|
||||
assert!(
|
||||
@@ -1750,7 +1729,7 @@ mod serial_tests {
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#
|
||||
);
|
||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
|
||||
@@ -1815,12 +1794,11 @@ mod serial_tests {
|
||||
</NoncurrentVersionExpiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#;
|
||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
|
||||
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
|
||||
.await
|
||||
.expect("Failed to set noncurrent lifecycle configuration");
|
||||
|
||||
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
|
||||
.await;
|
||||
init_background_expiry(ecstore.clone()).await;
|
||||
|
||||
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||
|
||||
@@ -1854,7 +1832,7 @@ mod serial_tests {
|
||||
</NoncurrentVersionExpiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#;
|
||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
|
||||
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
|
||||
.await
|
||||
.expect("Failed to set noncurrent lifecycle configuration");
|
||||
|
||||
@@ -1941,7 +1919,7 @@ mod serial_tests {
|
||||
</Rule>
|
||||
</LifecycleConfiguration>"#
|
||||
);
|
||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CapabilityState {
|
||||
Supported,
|
||||
Unsupported,
|
||||
Disabled,
|
||||
#[default]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl CapabilityState {
|
||||
pub const fn is_supported(self) -> bool {
|
||||
matches!(self, Self::Supported)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CapabilityStatus {
|
||||
pub state: CapabilityState,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reason: Option<String>,
|
||||
}
|
||||
|
||||
impl CapabilityStatus {
|
||||
pub const fn new(state: CapabilityState) -> Self {
|
||||
Self { state, reason: None }
|
||||
}
|
||||
|
||||
pub const fn supported() -> Self {
|
||||
Self::new(CapabilityState::Supported)
|
||||
}
|
||||
|
||||
pub const fn unsupported() -> Self {
|
||||
Self::new(CapabilityState::Unsupported)
|
||||
}
|
||||
|
||||
pub const fn disabled() -> Self {
|
||||
Self::new(CapabilityState::Disabled)
|
||||
}
|
||||
|
||||
pub const fn unknown() -> Self {
|
||||
Self::new(CapabilityState::Unknown)
|
||||
}
|
||||
|
||||
pub fn with_reason(mut self, reason: impl Into<String>) -> Self {
|
||||
self.reason = Some(reason.into());
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CapabilityStatus {
|
||||
fn default() -> Self {
|
||||
Self::unknown()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CapabilitySnapshotError {
|
||||
Unavailable,
|
||||
Unsupported,
|
||||
InvalidSnapshot(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for CapabilitySnapshotError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Unavailable => f.write_str("capability snapshot unavailable"),
|
||||
Self::Unsupported => f.write_str("capability snapshot unsupported"),
|
||||
Self::InvalidSnapshot(reason) => write!(f, "invalid capability snapshot: {reason}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for CapabilitySnapshotError {}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn capability_status_serializes_unknown_and_unsupported_states() {
|
||||
let unknown = CapabilityStatus::unknown();
|
||||
let unsupported = CapabilityStatus::unsupported().with_reason("target does not expose profiler");
|
||||
|
||||
let encoded = serde_json::to_string(&(unknown, unsupported)).expect("serialize capability statuses");
|
||||
let decoded: (CapabilityStatus, CapabilityStatus) =
|
||||
serde_json::from_str(&encoded).expect("deserialize capability statuses");
|
||||
|
||||
assert_eq!(decoded.0.state, CapabilityState::Unknown);
|
||||
assert_eq!(decoded.1.state, CapabilityState::Unsupported);
|
||||
assert_eq!(decoded.1.reason.as_deref(), Some("target does not expose profiler"));
|
||||
assert!(!decoded.1.state.is_supported());
|
||||
}
|
||||
}
|
||||
@@ -16,12 +16,16 @@
|
||||
|
||||
pub mod admin;
|
||||
pub mod bucket;
|
||||
pub mod capability;
|
||||
pub mod error;
|
||||
pub mod multipart;
|
||||
pub mod object;
|
||||
pub mod observability;
|
||||
pub mod topology;
|
||||
|
||||
pub use admin::{DiskSetSelector, StorageAdminApi};
|
||||
pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
|
||||
pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
|
||||
pub use error::{StorageErrorCode, StorageResult};
|
||||
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
|
||||
pub use object::{DeletedObject, ObjectToDelete};
|
||||
@@ -31,3 +35,10 @@ pub use object::{HealOperations, MultipartOperations, NamespaceLocking, ObjectIO
|
||||
pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};
|
||||
pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};
|
||||
pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};
|
||||
pub use observability::{
|
||||
MemorySamplingState, ObservabilitySnapshot, ObservabilitySnapshotProvider, PlatformSupport, UserspaceProfilingCapability,
|
||||
};
|
||||
pub use topology::{
|
||||
DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet, TopologySnapshot,
|
||||
TopologySnapshotProvider,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::fmt::Debug;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{CapabilitySnapshotError, CapabilityStatus};
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ObservabilitySnapshot {
|
||||
pub runtime_telemetry: CapabilityStatus,
|
||||
pub userspace_profiling: UserspaceProfilingCapability,
|
||||
pub memory_sampling: MemorySamplingState,
|
||||
pub platform: PlatformSupport,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct UserspaceProfilingCapability {
|
||||
pub cpu: CapabilityStatus,
|
||||
pub memory: CapabilityStatus,
|
||||
pub continuous_cpu: CapabilityStatus,
|
||||
pub periodic_cpu: CapabilityStatus,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MemorySamplingState {
|
||||
pub process: CapabilityStatus,
|
||||
pub system: CapabilityStatus,
|
||||
pub cgroup: CapabilityStatus,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PlatformSupport {
|
||||
pub target_triple: Option<String>,
|
||||
pub os: Option<String>,
|
||||
pub arch: Option<String>,
|
||||
pub allocator: CapabilityStatus,
|
||||
pub ebpf: CapabilityStatus,
|
||||
pub numa: CapabilityStatus,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait ObservabilitySnapshotProvider: Send + Sync + Debug {
|
||||
async fn observability_snapshot(&self) -> Result<ObservabilitySnapshot, CapabilitySnapshotError>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::CapabilityState;
|
||||
|
||||
#[test]
|
||||
fn observability_snapshot_preserves_unknown_and_unsupported_states() {
|
||||
let snapshot = ObservabilitySnapshot {
|
||||
runtime_telemetry: CapabilityStatus::unknown(),
|
||||
userspace_profiling: UserspaceProfilingCapability {
|
||||
cpu: CapabilityStatus::unsupported().with_reason("unsupported target"),
|
||||
memory: CapabilityStatus::disabled(),
|
||||
continuous_cpu: CapabilityStatus::unknown(),
|
||||
periodic_cpu: CapabilityStatus::supported(),
|
||||
},
|
||||
memory_sampling: MemorySamplingState {
|
||||
process: CapabilityStatus::supported(),
|
||||
system: CapabilityStatus::supported(),
|
||||
cgroup: CapabilityStatus::unknown(),
|
||||
},
|
||||
platform: PlatformSupport {
|
||||
target_triple: Some("x86_64-unknown-linux-gnu".to_owned()),
|
||||
os: Some("linux".to_owned()),
|
||||
arch: Some("x86_64".to_owned()),
|
||||
allocator: CapabilityStatus::supported(),
|
||||
ebpf: CapabilityStatus::unknown(),
|
||||
numa: CapabilityStatus::unsupported(),
|
||||
},
|
||||
};
|
||||
|
||||
let encoded = serde_json::to_string(&snapshot).expect("serialize observability snapshot");
|
||||
let decoded: ObservabilitySnapshot = serde_json::from_str(&encoded).expect("deserialize observability snapshot");
|
||||
|
||||
assert_eq!(decoded.runtime_telemetry.state, CapabilityState::Unknown);
|
||||
assert_eq!(decoded.userspace_profiling.cpu.state, CapabilityState::Unsupported);
|
||||
assert_eq!(decoded.userspace_profiling.memory.state, CapabilityState::Disabled);
|
||||
assert_eq!(decoded.platform.numa.state, CapabilityState::Unsupported);
|
||||
assert_eq!(decoded.platform.ebpf.state, CapabilityState::Unknown);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt::Debug;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{CapabilitySnapshotError, CapabilityStatus};
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TopologySnapshot {
|
||||
pub pools: Vec<TopologyPool>,
|
||||
pub capabilities: TopologyCapabilities,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TopologyCapabilities {
|
||||
pub profiling: CapabilityStatus,
|
||||
pub numa: CapabilityStatus,
|
||||
pub failure_domain_labels: CapabilityStatus,
|
||||
pub media_labels: CapabilityStatus,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TopologyPool {
|
||||
pub pool_index: usize,
|
||||
pub pool_id: Option<String>,
|
||||
pub labels: TopologyLabels,
|
||||
pub sets: Vec<TopologySet>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TopologySet {
|
||||
pub pool_index: usize,
|
||||
pub set_index: usize,
|
||||
pub set_id: Option<String>,
|
||||
pub labels: TopologyLabels,
|
||||
pub disks: Vec<TopologyDisk>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TopologyDisk {
|
||||
pub pool_index: usize,
|
||||
pub set_index: usize,
|
||||
pub disk_index: usize,
|
||||
pub disk_id: Option<String>,
|
||||
pub labels: TopologyLabels,
|
||||
pub capabilities: DiskCapabilities,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TopologyLabels {
|
||||
pub zone: Option<String>,
|
||||
pub rack: Option<String>,
|
||||
pub node: Option<String>,
|
||||
pub media: Option<String>,
|
||||
pub numa_node: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub additional: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct DiskCapabilities {
|
||||
pub media_type: CapabilityStatus,
|
||||
pub failure_domain: CapabilityStatus,
|
||||
pub numa: CapabilityStatus,
|
||||
pub profiling: CapabilityStatus,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait TopologySnapshotProvider: Send + Sync + Debug {
|
||||
async fn topology_snapshot(&self) -> Result<TopologySnapshot, CapabilitySnapshotError>;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::CapabilityState;
|
||||
|
||||
#[test]
|
||||
fn topology_snapshot_allows_missing_and_extra_labels() {
|
||||
let raw = r#"{
|
||||
"pools": [{
|
||||
"pool_index": 0,
|
||||
"pool_id": null,
|
||||
"labels": {
|
||||
"additional": {
|
||||
"room": "a"
|
||||
}
|
||||
},
|
||||
"sets": [{
|
||||
"pool_index": 0,
|
||||
"set_index": 1,
|
||||
"set_id": null,
|
||||
"labels": {},
|
||||
"disks": [{
|
||||
"pool_index": 0,
|
||||
"set_index": 1,
|
||||
"disk_index": 2,
|
||||
"disk_id": "disk-2",
|
||||
"labels": {
|
||||
"media": "ssd",
|
||||
"additional": {
|
||||
"slot": "nvme0"
|
||||
}
|
||||
},
|
||||
"capabilities": {
|
||||
"media_type": { "state": "supported" },
|
||||
"failure_domain": { "state": "unknown" },
|
||||
"numa": { "state": "unsupported", "reason": "not reported" },
|
||||
"profiling": { "state": "disabled" }
|
||||
}
|
||||
}]
|
||||
}]
|
||||
}],
|
||||
"capabilities": {
|
||||
"profiling": { "state": "supported" },
|
||||
"numa": { "state": "unknown" },
|
||||
"failure_domain_labels": { "state": "supported" },
|
||||
"media_labels": { "state": "supported" }
|
||||
}
|
||||
}"#;
|
||||
|
||||
let snapshot: TopologySnapshot = serde_json::from_str(raw).expect("deserialize topology snapshot");
|
||||
let disk = &snapshot.pools[0].sets[0].disks[0];
|
||||
|
||||
assert_eq!(snapshot.pools[0].labels.zone, None);
|
||||
assert_eq!(snapshot.pools[0].labels.additional.get("room").map(String::as_str), Some("a"));
|
||||
assert_eq!(disk.labels.media.as_deref(), Some("ssd"));
|
||||
assert_eq!(disk.labels.additional.get("slot").map(String::as_str), Some("nvme0"));
|
||||
assert_eq!(disk.capabilities.numa.state, CapabilityState::Unsupported);
|
||||
assert_eq!(disk.capabilities.profiling.state, CapabilityState::Disabled);
|
||||
}
|
||||
}
|
||||
@@ -92,14 +92,17 @@ Required `rustfs-storage-api` public re-exports:
|
||||
|
||||
- `pub use admin::{DiskSetSelector, StorageAdminApi};`
|
||||
- `pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};`
|
||||
- `pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};`
|
||||
- `pub use error::{StorageErrorCode, StorageResult};`
|
||||
- `pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};`
|
||||
- `pub use observability::{MemorySamplingState, ObservabilitySnapshot, ObservabilitySnapshotProvider, PlatformSupport, UserspaceProfilingCapability};`
|
||||
- `pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};`
|
||||
- `pub use object::{ExpirationOptions, TransitionedObject};`
|
||||
- `pub use object::{HealOperations, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations};`
|
||||
- `pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};`
|
||||
- `pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};`
|
||||
- `pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};`
|
||||
- `pub use topology::{DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet, TopologySnapshot, TopologySnapshotProvider};`
|
||||
|
||||
ECStore must keep compile-time coverage for `StorageAdminApi`, `HealOperations`,
|
||||
and the separate `NamespaceLocking` operation group.
|
||||
|
||||
@@ -5,17 +5,19 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
## Current Context
|
||||
|
||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||
- Branch: `overtrue/arch-storage-api-lifecycle-contracts`
|
||||
- Baseline: stacked on `rustfs/rustfs#3594` head
|
||||
(`6fc05b84e2cd22a0482adfbc042184b03f8fdfa6`).
|
||||
- PR type for this branch: `pure-move`
|
||||
- Runtime behavior changes: no migration behavior change expected.
|
||||
- Rust code changes: move lifecycle helper DTO contracts for expiration and
|
||||
transitioned object metadata into rustfs-storage-api, switch ECStore internal
|
||||
consumers to direct storage-api imports, and keep ECStore old-path re-exports.
|
||||
- CI/script changes: add migration guards rejecting reintroduced ECStore
|
||||
lifecycle helper DTO definitions and old internal consumer imports.
|
||||
- Docs changes: record the lifecycle helper pure-move slice.
|
||||
- Branch: `overtrue/arch-observability-topology-contracts`
|
||||
- Baseline: stacked on `origin/overtrue/arch-test-harness-compat-aliases`
|
||||
after `rustfs/rustfs#3600`
|
||||
(`ae6a5befcf60f2f2ebce9799ba93649032234273`).
|
||||
- PR type for this branch: `contract`
|
||||
- Runtime behavior changes: none.
|
||||
- Rust code changes: add observability and topology capability DTO/trait
|
||||
contracts to `rustfs-storage-api`.
|
||||
- CI/script changes: extend migration re-export guard coverage for the new
|
||||
contract exports.
|
||||
- Docs changes: add
|
||||
[`runtime-capability-contracts.md`](runtime-capability-contracts.md) and
|
||||
record the combined PR-08/API-013 plus PR-09/API-014 contract slice.
|
||||
|
||||
## Phase 0 Tasks
|
||||
|
||||
@@ -63,6 +65,60 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
[`ecstore-config-consumer-inventory.md`](ecstore-config-consumer-inventory.md)
|
||||
records the current model definitions, global accessors, persistence helpers,
|
||||
consumer groups, migration risks, and do-not-change contract.
|
||||
- [x] `G-011` Inventory scheduler baseline.
|
||||
- Acceptance:
|
||||
[`scheduler-baseline.md`](scheduler-baseline.md) records current owners for
|
||||
request admission, reusable scheduler/backpressure facades, workers, scanner
|
||||
budget, heal admission, and the Tokio runtime builder.
|
||||
- Must preserve: no Rust source changes, no scheduler/controller contract
|
||||
changes, and no runtime behavior changes.
|
||||
- [x] `G-012` Inventory placement and repair invariants.
|
||||
- Acceptance:
|
||||
[`placement-repair-invariants.md`](placement-repair-invariants.md) records
|
||||
object-to-set hashing, pool/set/disk assignment boundaries, set-aware
|
||||
readiness and lock quorum, scanner budget, and heal admission preservation
|
||||
gates.
|
||||
- Must preserve: no placement, repair, scanner, heal, readiness, lock, or
|
||||
storage metadata behavior changes.
|
||||
- [x] `G-013` Inventory profiling and NUMA capabilities.
|
||||
- Acceptance:
|
||||
[`profiling-numa-capability-inventory.md`](profiling-numa-capability-inventory.md)
|
||||
records current CPU/memory profiling, cgroup memory sampling, allocator
|
||||
backend, eBPF, and NUMA capability support plus no-op fallback invariants.
|
||||
- Must preserve: no startup, profiling, allocator, runtime, or platform-gate
|
||||
behavior changes.
|
||||
|
||||
## Issue #660 Capability Contract Tasks
|
||||
|
||||
- [x] `PR-08/API-013` Add observability snapshot contract.
|
||||
- Completed slice: add `CapabilityState`, `CapabilityStatus`,
|
||||
`CapabilitySnapshotError`, `ObservabilitySnapshot`,
|
||||
`UserspaceProfilingCapability`, `MemorySamplingState`,
|
||||
`PlatformSupport`, and `ObservabilitySnapshotProvider` to
|
||||
`rustfs-storage-api`.
|
||||
- Acceptance: runtime telemetry, userspace profiling, memory sampling, and
|
||||
platform support states are representable without runtime, ECStore, admin,
|
||||
profiling, exporter, sidecar, eBPF, or OTEL implementation dependencies.
|
||||
- Must preserve: no profiling, startup, admin route, exporter, sidecar, eBPF,
|
||||
OTEL, or runtime behavior changes.
|
||||
- Verification: storage-api contract tests for unknown, unsupported,
|
||||
disabled, and supported capability states; focused storage-api check;
|
||||
migration guard; formatting; diff hygiene; and three-expert review.
|
||||
|
||||
- [x] `PR-09/API-014` Add topology capability contract.
|
||||
- Completed slice: add `TopologySnapshot`, `TopologyCapabilities`,
|
||||
`TopologyPool`, `TopologySet`, `TopologyDisk`, `TopologyLabels`,
|
||||
`DiskCapabilities`, and `TopologySnapshotProvider` to
|
||||
`rustfs-storage-api`.
|
||||
- Acceptance: pool, set, and disk identity fields plus optional zone, rack,
|
||||
node, media, NUMA, and additional labels are representable without
|
||||
`rustfs-ecstore`.
|
||||
- Must preserve: no ECStore endpoint/set implementation, placement,
|
||||
membership, NUMA pinning, or runtime behavior changes.
|
||||
- Verification: storage-api contract tests for missing and additional labels
|
||||
plus supported, unsupported, unknown, and disabled capability states;
|
||||
focused storage-api check; migration guard; formatting; diff hygiene; and
|
||||
three-expert review.
|
||||
- [x] `TEST-PRTYPE-001` Check PR type enum consistency.
|
||||
- Acceptance: `./scripts/check_architecture_migration_rules.sh` parses the
|
||||
allowed PR types from [`crate-boundaries.md`](crate-boundaries.md) and fails
|
||||
@@ -1296,6 +1352,87 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
migration and layer guards, formatting check, diff hygiene, risk scan, full
|
||||
pre-commit, and required three-expert review passed before push.
|
||||
|
||||
- [x] `API-051` Flatten test harness storage compatibility aliases.
|
||||
- Current branch: `overtrue/arch-test-harness-compat-aliases`.
|
||||
- Current slice: flatten e2e, heal, scanner, and fuzz storage compatibility
|
||||
harnesses from nested `storage_compat::ecstore` modules into direct
|
||||
crate-local aliases, constants, and function imports.
|
||||
- Acceptance: no e2e, heal-test, scanner-test, or fuzz-target harness file
|
||||
may expose or consume nested `storage_compat::ecstore` paths, and migration
|
||||
rules reject reintroducing nested test/fuzz ECStore compatibility modules.
|
||||
- Must preserve: e2e bucket target/RPC/disk helper imports, heal ECStore disk
|
||||
and endpoint setup, scanner lifecycle/tier/disk/storage setup, fuzz bucket
|
||||
validation behavior, and fuzz path-containment validation behavior.
|
||||
- Risk defense: this is test-harness and fuzz-harness import cleanup only; no
|
||||
production runtime behavior, ECStore ownership, storage metadata format, or
|
||||
scanner/heal lifecycle logic is changed.
|
||||
- Verification: focused e2e/heal/scanner test compile, harness tests,
|
||||
migration and layer guards, formatting check, diff hygiene, risk scan, full
|
||||
pre-commit, and required three-expert review passed before push.
|
||||
|
||||
- [x] `API-052` Flatten RustFS runtime storage compatibility aliases.
|
||||
- Current branch: `overtrue/arch-rustfs-storage-compat-aliases`.
|
||||
- Current slice: flatten RustFS root, app, admin, and storage runtime
|
||||
compatibility facades from nested `storage_compat::ecstore` modules into
|
||||
direct crate-local aliases, constants, and function imports.
|
||||
- Acceptance: no RustFS runtime source file may expose or consume nested
|
||||
`storage_compat::ecstore` paths, and migration rules reject reintroducing
|
||||
nested RustFS runtime ECStore compatibility modules.
|
||||
- Must preserve: startup/config/bootstrap behavior, server readiness checks,
|
||||
admin replication/rebalance/tier/config handlers, app object/bucket/
|
||||
multipart usecases, storage RPC/SSE/access paths, table catalog storage
|
||||
access, and existing local compatibility ownership.
|
||||
- Risk defense: this is RustFS runtime import cleanup only; no production
|
||||
runtime behavior, ECStore ownership, storage metadata format, object I/O,
|
||||
admin authorization, or readiness semantics are changed.
|
||||
- Verification: focused RustFS compile, migration and layer guards,
|
||||
formatting check, diff hygiene, risk scan, full pre-commit, and required
|
||||
three-expert review passed before push.
|
||||
|
||||
- [x] `API-053` Flatten RustFS runtime scalar storage compatibility aliases.
|
||||
- Current branch: `overtrue/arch-runtime-compat-surface-prune`.
|
||||
- Current slice: flatten RustFS root, app, admin, and storage runtime scalar
|
||||
compatibility facades such as store, error, global, endpoints, RPC,
|
||||
metrics, notification, set-disk, and data-usage paths into direct
|
||||
crate-local aliases and functions.
|
||||
- Acceptance: RustFS runtime source no longer consumes those scalar
|
||||
compatibility surfaces through secondary modules, while higher-coupling
|
||||
bucket/config/rio compatibility modules remain unchanged; migration rules
|
||||
reject restoring the flattened scalar paths.
|
||||
- Must preserve: startup config/bootstrap behavior, server readiness checks,
|
||||
admin replication/rebalance/tier/config handlers, app object/bucket/
|
||||
multipart usecases, storage RPC/SSE/access paths, table catalog storage
|
||||
access, and existing ECStore concrete type ownership.
|
||||
- Risk defense: this is import ownership and facade-shape cleanup only; no
|
||||
production runtime behavior, ECStore ownership, storage metadata format,
|
||||
object I/O, admin authorization, or readiness semantics are changed.
|
||||
- Verification: focused RustFS compile, migration and layer guards,
|
||||
formatting check, diff hygiene, risk scan, full pre-commit, and required
|
||||
three-expert review passed before push.
|
||||
|
||||
- [x] `API-054` Flatten RustFS runtime secondary storage compatibility aliases.
|
||||
- Current branch: `overtrue/arch-runtime-secondary-compat-flatten`.
|
||||
- Current slice: flatten RustFS root, app, admin, and storage runtime
|
||||
secondary compatibility modules such as bucket, config, rio, client, tier,
|
||||
compress, disk, and rebalance into direct crate-local aliases, modules, and
|
||||
functions.
|
||||
- Acceptance: RustFS runtime source no longer consumes those compatibility
|
||||
surfaces through broad secondary modules, the runtime compatibility files no
|
||||
longer define those wrapper modules, and migration rules reject restoring
|
||||
the flattened secondary paths.
|
||||
- Must preserve: startup config/bootstrap behavior, server module-switch
|
||||
config reads, embedded startup storage initialization, admin bucket/meta/
|
||||
tier/rebalance/config handlers, app object/bucket/multipart usecases,
|
||||
storage RPC/SSE/access paths, table catalog storage access, and ECStore
|
||||
concrete type ownership.
|
||||
- Risk defense: this is import ownership and facade-shape cleanup only; no
|
||||
production runtime behavior, ECStore ownership, storage metadata format,
|
||||
object I/O, admin authorization, tier behavior, or readiness semantics are
|
||||
changed.
|
||||
- Verification: focused RustFS compile, migration and layer guards,
|
||||
formatting check, diff hygiene, risk scan, and required three-expert review
|
||||
passed before push.
|
||||
|
||||
## Phase 8 Background Controller Tasks
|
||||
|
||||
- [x] `BGC-001` Inventory background services.
|
||||
@@ -1564,7 +1701,10 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
|
||||
## Next PRs
|
||||
|
||||
1. `pure-move`/`consumer-migration`: continue larger cleanup slices with the
|
||||
1. `contract`/`consumer-migration`: wire read-only observability and topology
|
||||
snapshots to implementation owners without changing runtime, profiling,
|
||||
placement, or admin route behavior.
|
||||
2. `pure-move`/`consumer-migration`: continue larger cleanup slices with the
|
||||
loss-prevention guards active for remaining ECStore compatibility contracts
|
||||
now that broad compatibility passthroughs are fully closed.
|
||||
|
||||
@@ -1572,14 +1712,65 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
|
||||
| Expert | Status | Notes |
|
||||
|---|---|---|
|
||||
| Quality/architecture | passed | S-015 removes obsolete KMS admin policy action variants after the handler fallback cleanup; API-042/API-043/API-044/API-045/API-046/API-047/API-048/API-049/API-050 narrow notify, S3 Select, OBS, IAM, Swift, heal, scanner, RustFS runtime, test, fuzz, and lifecycle helper compatibility contracts without moving ECStore storage metadata ownership. |
|
||||
| Migration preservation | passed | KMS endpoint URLs, query aliases, request bodies, response contracts, and dedicated `kms:*` authorization behavior are preserved; event builder call sites, ECStore event bridge conversion, restore event data, version IDs, metadata filtering, config read/save semantics, S3 Select store/error/buffer semantics, OBS metrics state reads, IAM config/notification/error semantics, Swift bucket metadata access, heal disk/resume/task behavior, scanner lifecycle/replication/data-usage behavior, RustFS startup/admin/app/storage runtime access, e2e/test/fuzz import behavior, lifecycle expiration/transition helper DTO field contracts, unchanged no-op handling, and remove-event behavior are preserved. |
|
||||
| Testing/verification | passed | Focused compiles/tests, guards, formatting, diff hygiene, risk scan, and full `make pre-commit` passed for the current slice. |
|
||||
| Quality/architecture | passed | S-015 removes obsolete KMS admin policy action variants after the handler fallback cleanup; API-042/API-043/API-044/API-045/API-046/API-047/API-048/API-049/API-050/API-051/API-052/API-053/API-054 narrow notify, S3 Select, OBS, IAM, Swift, heal, scanner, RustFS runtime, test, fuzz, lifecycle helper, harness, and RustFS runtime compatibility contracts without moving ECStore storage metadata ownership; G-011/G-012/G-013 add docs-only baselines for scheduler, placement/repair, and profiling/NUMA work; Issue #660 PR-08/PR-09 add read-only observability and topology contracts in rustfs-storage-api only. |
|
||||
| Migration preservation | passed | KMS endpoint URLs, query aliases, request bodies, response contracts, and dedicated `kms:*` authorization behavior are preserved; event builder call sites, ECStore event bridge conversion, restore event data, version IDs, metadata filtering, config read/save semantics, S3 Select store/error/buffer semantics, OBS metrics state reads, IAM config/notification/error semantics, Swift bucket metadata access, heal disk/resume/task behavior, scanner lifecycle/replication/data-usage behavior, RustFS startup/admin/app/storage runtime access, e2e/test/fuzz import behavior, lifecycle expiration/transition helper DTO field contracts, flattened harness and RustFS runtime scalar/secondary alias behavior, unchanged no-op handling, remove-event behavior, scheduler/readiness/placement/profiling runtime behavior, platform gates, missing/unknown capability states, and placement/topology labels are preserved. |
|
||||
| Testing/verification | passed | Focused compiles/tests, fuzz target compile, guards, formatting, diff hygiene, risk scan, and full `make pre-commit` passed for prior code slices; current Issue #660 PR-08/PR-09 contract slice uses storage-api tests/checks, migration guard, formatting, diff hygiene, and three-expert review. |
|
||||
|
||||
## Verification Notes
|
||||
|
||||
Passed before push:
|
||||
|
||||
- Issue #660 PR-08/PR-09 current slice:
|
||||
- `cargo test -p rustfs-storage-api`: passed.
|
||||
- `cargo check -p rustfs-storage-api`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- `make pre-commit`: passed.
|
||||
- Three-expert review: passed.
|
||||
|
||||
- G-011/G-012/G-013 current slice:
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- Three-expert review: passed.
|
||||
- Full `make pre-commit`: not run because this slice is documentation-only.
|
||||
|
||||
- API-054 current slice:
|
||||
- `cargo check -p rustfs --lib`: passed.
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- Rust risk scan: passed; only existing import and path rewrites were
|
||||
reviewed, with no new unwrap/expect, panic/todo/unsafe, risky casts,
|
||||
ad-hoc error construction, or sensitive-token handling semantics.
|
||||
|
||||
- API-053 current slice:
|
||||
- `cargo check -p rustfs --lib`: passed.
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- Rust risk scan: passed; only existing import and path rewrites were
|
||||
reviewed, with no new unwrap/expect, panic/todo/unsafe, risky casts,
|
||||
ad-hoc error construction, or sensitive-token handling semantics.
|
||||
- `make pre-commit`: passed.
|
||||
|
||||
- API-052 current slice:
|
||||
- `cargo check -p rustfs --lib`: passed.
|
||||
- `cargo check --tests -p rustfs`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- Rust risk scan: passed; only existing-semantic path replacement hits were
|
||||
reviewed, with no new unwrap/expect, panic/todo/unsafe, risky casts,
|
||||
ad-hoc error construction, or sensitive-token handling semantics.
|
||||
- `make pre-commit`: passed, including 6250 nextest tests and doctests.
|
||||
|
||||
- API-050 current slice:
|
||||
- `cargo test -p rustfs-storage-api lifecycle_helper_defaults_preserve_existing_contracts --no-fail-fast`:
|
||||
passed.
|
||||
@@ -1598,6 +1789,24 @@ Passed before push:
|
||||
lines.
|
||||
- `make pre-commit`: passed.
|
||||
|
||||
- API-051 current slice:
|
||||
- `cargo check --tests -p e2e_test -p rustfs-heal -p rustfs-scanner`:
|
||||
passed.
|
||||
- `cargo check --manifest-path fuzz/Cargo.toml --all-targets`: passed.
|
||||
- `cargo test -p rustfs-heal --test endpoint_index_test test_endpoint_index_settings --no-fail-fast`:
|
||||
passed.
|
||||
- `cargo test -p rustfs-scanner --test lifecycle_integration_test --no-run`:
|
||||
passed.
|
||||
- `cargo test -p e2e_test --no-run`: passed.
|
||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||
- `cargo fmt --all --check`: passed.
|
||||
- `git diff --check`: passed.
|
||||
- Rust risk scan: passed; only existing test `unwrap` calls were touched by
|
||||
import path rewrites, with no new unwrap/expect, panic/todo/unsafe, risky
|
||||
casts, ad-hoc error construction, or sensitive-token handling semantics.
|
||||
- `make pre-commit`: passed.
|
||||
|
||||
- S-015 current slice:
|
||||
- `cargo test -p rustfs-policy test_legacy_kms_admin_actions_are_rejected --no-fail-fast`:
|
||||
passed.
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
# Placement And Repair Invariants
|
||||
|
||||
This inventory covers `G-012` for `rustfs/backlog#666`. It records the current
|
||||
object placement, readiness, lock quorum, scanner, and repair boundaries that
|
||||
later scheduler or topology work must preserve.
|
||||
|
||||
## Object To Set Hash Rule
|
||||
|
||||
Objects reach a set through `Sets::get_disks_by_key`, which calls
|
||||
`get_hashed_set_index` on the object key:
|
||||
|
||||
- `DistributionAlgoVersion::V1` uses `crc_hash(input, set_count)`.
|
||||
- `DistributionAlgoVersion::V2` and `V3` use
|
||||
`sip_hash(input, set_count, format_id_bytes)`.
|
||||
- The format ID is part of the V2/V3 distribution seed, so changing the seed,
|
||||
object key, set count, or algorithm changes placement.
|
||||
|
||||
Preservation rule: every object read, write, list, heal, repair, and
|
||||
decommission path that resolves a set for an existing object must preserve the
|
||||
same object key and format distribution algorithm.
|
||||
|
||||
## Pool, Set, And Disk Assignment Boundary
|
||||
|
||||
Pool selection is separate from set hashing:
|
||||
|
||||
- Existing objects are discovered across pools and resolved to the best current
|
||||
pool candidate before reads or updates continue.
|
||||
- New object writes select an available pool from current per-pool free-space
|
||||
inputs after suspended or rebalancing pools are skipped.
|
||||
- Set selection inside a pool still uses the object-to-set hash rule above.
|
||||
- Disk index assignment comes from endpoint and format metadata, not from a
|
||||
scheduler decision.
|
||||
|
||||
Boundary rule: schedulers may influence admission, worker concurrency, or
|
||||
buffer sizing, but they must not rewrite pool, set, or disk indexes.
|
||||
|
||||
## Readiness And Lock Quorum Boundary
|
||||
|
||||
Runtime readiness currently checks storage and lock health independently:
|
||||
|
||||
- Storage readiness requires every observed set to meet write quorum based on
|
||||
the set drive count and storage class data/parity shape.
|
||||
- Lock readiness aggregates per-set lock-client host quorum and fails fast if
|
||||
any set loses quorum.
|
||||
- Object and bucket mutations acquire namespace locks through the existing
|
||||
storage lock wrappers before changing object or bucket state.
|
||||
|
||||
Boundary rule: readiness and lock quorum must stay set-aware. A global healthy
|
||||
disk count or global connected-host count is not sufficient when any individual
|
||||
set is below quorum.
|
||||
|
||||
## Scanner Budget Preservation
|
||||
|
||||
Scanner cycles are bounded by `ScannerCycleBudget`:
|
||||
|
||||
- Runtime budget cancels the child token after the configured duration.
|
||||
- Object budget cancels after the configured object count.
|
||||
- Directory budget rejects additional directories and cancels with the
|
||||
directories reason.
|
||||
- Partial-cycle metrics and checkpoints use the budget reason.
|
||||
|
||||
Preservation rule: later scheduler work can change how scan cycles are admitted
|
||||
only if it preserves the budget reason, checkpoint reason, and child-token
|
||||
cancellation behavior.
|
||||
|
||||
## Heal Admission Preservation
|
||||
|
||||
Scanner and background repair work enter the heal manager through explicit
|
||||
admission:
|
||||
|
||||
- Scanner object heal requests are low priority and may be accepted, merged,
|
||||
rejected as full, or dropped.
|
||||
- Required/high-priority heal candidates escalate on non-admission instead of
|
||||
silently disappearing.
|
||||
- Heal queue admission deduplicates queued and active work unless the request
|
||||
explicitly forces admission.
|
||||
- Full queues can drop low-priority work or displace lower-priority work for a
|
||||
higher-priority request according to current manager rules.
|
||||
|
||||
Preservation rule: repair scheduling changes must keep admission outcomes
|
||||
observable and must not convert rejected or dropped repair work into silent
|
||||
success.
|
||||
|
||||
## Behavior Change Gates
|
||||
|
||||
Any later placement or repair PR must use the following gates:
|
||||
|
||||
- Placement gate: prove object-to-set hashing is unchanged for existing object
|
||||
keys and format algorithms.
|
||||
- Pool gate: prove pool selection does not choose suspended or rebalancing
|
||||
pools unless the existing path already allows it.
|
||||
- Quorum gate: prove storage readiness and lock readiness remain per-set.
|
||||
- Scanner gate: prove scan budget reason and checkpoint mapping remain stable.
|
||||
- Heal gate: prove low-priority scanner heal, forced heal, duplicate merge, and
|
||||
queue-full outcomes remain distinct.
|
||||
- Rollback gate: if a new scheduler sidecar is disabled, placement and repair
|
||||
must fall back to the current direct ECStore/scanner/heal behavior.
|
||||
@@ -0,0 +1,81 @@
|
||||
# Profiling And NUMA Capability Inventory
|
||||
|
||||
This inventory covers `G-013` for `rustfs/backlog#667`. It records the current
|
||||
profiling, memory sampling, allocator, and NUMA baseline before optional runtime
|
||||
sidecars are designed.
|
||||
|
||||
## Platform Support Matrix
|
||||
|
||||
| Capability | Current support | Current owner | Baseline recommendation |
|
||||
|---|---|---|---|
|
||||
| CPU pprof dump | Linux and macOS builds use `pprof`; other targets return an unsupported-platform error. | `rustfs/src/profiling.rs` | Keep CPU profiling opt-in through existing env flags and cancellation token. |
|
||||
| Continuous CPU profiling | Linux and macOS builds can hold a continuous `ProfilerGuard` when enabled. | `rustfs/src/profiling.rs` | Preserve single-guard ownership and avoid starting multiple continuous guards. |
|
||||
| Periodic CPU profiling | Linux and macOS builds can spawn a periodic sampling loop. | `rustfs/src/profiling.rs` | Keep the loop cancellation-driven and non-fatal. |
|
||||
| Jemalloc memory pprof | Only `linux` + `gnu` + `x86_64` exposes jemalloc pprof dumping. Other supported builds return an unsupported-target error. | `rustfs/src/profiling.rs` | Treat memory pprof as optional and target-gated. |
|
||||
| Periodic memory pprof | Only runs where jemalloc profiling control is available and active. | `rustfs/src/profiling.rs` | Keep inactive jemalloc as a skipped dump, not a startup failure. |
|
||||
| Process/system memory sampling | Uses `rustfs_io_metrics::snapshot_process_resource_and_system` plus `sysinfo` total memory. | `rustfs/src/memory_observability.rs` | Keep sampling portable and metric-gated. |
|
||||
| cgroup memory sampling | Reads Linux cgroup v2 or v1 memory files when present. Missing files produce no cgroup split. | `rustfs/src/memory_observability.rs` | Keep cgroup data opportunistic and absent-safe. |
|
||||
| Allocator reclaim | Uses jemalloc backend on `linux` + `gnu` + `x86_64`; otherwise mimalloc variants. | `rustfs/src/allocator_reclaim.rs` | Keep backend detection read-only and preserve effective-force behavior. |
|
||||
| eBPF | No runtime eBPF sidecar is currently wired into startup. | N/A | Treat eBPF as future optional Linux-only inventory, never as a required baseline. |
|
||||
| NUMA | No NUMA placement or topology controller is currently wired into startup. | N/A | Treat NUMA as future optional capability with no-op fallback. |
|
||||
|
||||
## Cross-Platform Baseline
|
||||
|
||||
The current safe baseline is:
|
||||
|
||||
- Profiling is opt-in through env flags and must not make startup fatal.
|
||||
- Unsupported profiling targets return structured unsupported errors or skip
|
||||
startup tasks.
|
||||
- Memory observability records process/system metrics and adds cgroup split
|
||||
only when cgroup files exist.
|
||||
- Allocator reclaim observes active HTTP, delete-tail, scanner, heal, erasure,
|
||||
and GET-buffer activity before reclaiming.
|
||||
- Runtime thread sizing remains owned by the Tokio runtime builder and sysinfo
|
||||
core detection, not NUMA topology.
|
||||
|
||||
## Optional Sidecar Invariants
|
||||
|
||||
Future sidecars for profiling, eBPF, or NUMA must preserve these invariants:
|
||||
|
||||
- Sidecars must be disabled by default or target-gated until explicitly enabled.
|
||||
- Unsupported targets must degrade to no-op status, not panic or fail startup.
|
||||
- Sidecars must use the runtime cancellation token or an equivalent explicit
|
||||
shutdown handle.
|
||||
- Sidecars must not mutate Tokio worker counts after runtime creation.
|
||||
- Profiling output directory fallback must stay local to profiling and must not
|
||||
affect object storage paths.
|
||||
- NUMA fallback must preserve current runtime thread defaults, storage set
|
||||
placement, and request admission behavior.
|
||||
|
||||
## First Implementation Candidates
|
||||
|
||||
`API-013`:
|
||||
|
||||
- Define a read-only capability contract for profiling, cgroup memory, eBPF,
|
||||
allocator backend, and NUMA availability.
|
||||
- Keep the contract in a low-dependency crate and report unsupported states
|
||||
explicitly.
|
||||
|
||||
`R-016`:
|
||||
|
||||
- Wire storage runtime startup to consume capability snapshots read-only.
|
||||
- Do not start sidecars or mutate runtime worker ownership in the same PR.
|
||||
|
||||
`X-012`:
|
||||
|
||||
- Add an optional profiling/eBPF extension point that can observe capability
|
||||
status.
|
||||
- Keep unsupported targets and disabled sidecars as no-op.
|
||||
|
||||
`X-013`:
|
||||
|
||||
- Add extension tests for disabled, unsupported, and enabled capability
|
||||
snapshots.
|
||||
- Verify no startup fatal boundary is added for optional profiling sidecars.
|
||||
|
||||
`R-017`:
|
||||
|
||||
- If a runtime service sidecar is added later, start it from the startup service
|
||||
boundary with explicit shutdown ownership.
|
||||
- Preserve current service order, KMS/audit/notification fatal boundaries, and
|
||||
scanner/heal startup semantics.
|
||||
@@ -0,0 +1,44 @@
|
||||
# Runtime Capability Contracts
|
||||
|
||||
This document records the `rustfs/backlog#660` PR-08 and PR-09 contract slice.
|
||||
It adds read-only observability and topology snapshot shapes to
|
||||
`rustfs-storage-api` without coupling the contract crate to runtime, ECStore,
|
||||
admin routes, profiling, or observability implementation crates.
|
||||
|
||||
## Observability Snapshot Contract
|
||||
|
||||
`ObservabilitySnapshot` records:
|
||||
|
||||
- Runtime telemetry capability state.
|
||||
- Userspace CPU and memory profiling capability state.
|
||||
- Process, system, and cgroup memory sampling state.
|
||||
- Platform support for target triple, OS, architecture, allocator, eBPF, and
|
||||
NUMA capability.
|
||||
|
||||
Unsupported, disabled, and unknown states are represented by `CapabilityState`
|
||||
instead of failing snapshot construction. The contract is intentionally read
|
||||
only and does not replace existing profiling routes, telemetry APIs, exporter
|
||||
pipelines, or startup behavior.
|
||||
|
||||
## Topology Snapshot Contract
|
||||
|
||||
`TopologySnapshot` records:
|
||||
|
||||
- Pool, set, and disk identity indexes plus optional stable IDs.
|
||||
- Optional zone, rack, node, media, NUMA, and additional labels.
|
||||
- Topology-wide profiling, NUMA, failure-domain label, and media-label
|
||||
capability states.
|
||||
- Per-disk media, failure-domain, NUMA, and profiling capability states.
|
||||
|
||||
Missing labels are represented as absent `Option` values. Extra topology labels
|
||||
belong in the `additional` label map, so future inventory labels do not require
|
||||
ECStore type leakage.
|
||||
|
||||
## Boundary Rules
|
||||
|
||||
- No `rustfs-ecstore`, `rustfs-obs`, Axum, KMS, admin route, OTEL, eBPF, or
|
||||
profiling implementation dependency is added to `rustfs-storage-api`.
|
||||
- No placement, membership, NUMA pinning, profiling, startup, admin route, or
|
||||
exporter behavior changes are part of this contract slice.
|
||||
- Providers must map implementation failures into `CapabilitySnapshotError`
|
||||
before crossing the contract boundary.
|
||||
@@ -0,0 +1,78 @@
|
||||
# Scheduler Baseline Inventory
|
||||
|
||||
This inventory covers `G-011` for `rustfs/backlog#675`. It is a docs-only
|
||||
snapshot of the current scheduling, backpressure, worker, scanner, heal, and
|
||||
runtime-builder ownership. It does not define new behavior.
|
||||
|
||||
## Current Owners
|
||||
|
||||
| Surface | Current owner | Current responsibility | Migration boundary |
|
||||
|---|---|---|---|
|
||||
| `ConcurrencyManager` | `rustfs/src/storage/concurrency/manager.rs` | Owns the RustFS S3 read-path disk-read semaphore, I/O metrics, priority queue, storage media detection, access-pattern detection, and buffer strategy. | Keep request admission and I/O metrics behavior stable until a controller can consume the same state explicitly. |
|
||||
| `SchedulerManager` | `crates/concurrency/src/scheduler.rs` | Provides a reusable facade over `rustfs-io-core::IoScheduler` and derives buffer/priority decisions from `SchedulerPolicy`. | Treat it as reusable library surface; do not assume the RustFS S3 read path has already moved to this facade. |
|
||||
| `BackpressureManager` | `crates/concurrency/src/backpressure.rs` | Provides a reusable duplex-pipe backpressure facade over `rustfs-io-core::BackpressureMonitor`. | Keep pipe sizing and watermark policy separate from object-read disk semaphore admission. |
|
||||
| RustFS backpressure monitor | `rustfs/src/storage/backpressure.rs` | Tracks object-pipe watermark state used by RustFS storage backpressure tests and helpers. | Preserve current state labels and watermark semantics when consolidating with reusable facades. |
|
||||
| `Workers` | `crates/concurrency/src/workers.rs` | Provides cooperative worker-slot admission with `take`, `give`, and `wait`; current background workflows use it for bounded set workers. | Preserve blocking/wakeup semantics and over-release clamping. |
|
||||
| Scanner cycle budget | `crates/scanner/src/scanner_budget.rs` | Cancels a child token when runtime, object-count, or directory-count budget is reached. | Preserve partial-cycle reason mapping and checkpoint accounting. |
|
||||
| Heal admission | `crates/heal/src/heal/manager.rs`, `crates/heal/src/heal/channel.rs`, `rustfs_common::heal_channel` | Owns priority queue admission, duplicate merge/drop/full results, active-task tracking, retry admission, and channel responses. | Preserve low-priority scanner behavior and high-priority escalation gates. |
|
||||
| Tokio runtime builder | `rustfs/src/server/runtime.rs` | Builds the multi-thread runtime from env/defaults, sets thread counts, stack, queue/event intervals, I/O event cap, thread name, and optional dial9 tracing. | Keep runtime defaults and env names stable when later startup phases move ownership. |
|
||||
|
||||
## Current Flow
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
http["HTTP/S3 request"] --> app["app object usecase"]
|
||||
app --> guard["ConcurrencyManager::track_request"]
|
||||
app --> permit["disk-read semaphore permit"]
|
||||
permit --> strategy["I/O queue status and buffer strategy"]
|
||||
strategy --> ecstore["ECStore object/read path"]
|
||||
ecstore --> setdisks["hashed set disks"]
|
||||
|
||||
scanner["scanner cycle"] --> budget["ScannerCycleBudget"]
|
||||
budget --> folder["folder/object scan"]
|
||||
folder --> healreq["heal channel request"]
|
||||
healreq --> admission["HealManager admission queue"]
|
||||
admission --> healworker["heal workers and retries"]
|
||||
|
||||
startup["startup entrypoint"] --> runtime["Tokio runtime builder"]
|
||||
runtime --> services["background services"]
|
||||
```
|
||||
|
||||
## Missing State For Later Work
|
||||
|
||||
`R-015` storage foundation:
|
||||
|
||||
- Needs a stable inventory of endpoint publication, local disk prewarm, lock
|
||||
client setup, and per-set readiness state before any scheduler/controller
|
||||
consumes storage topology.
|
||||
- Must not infer set availability only from request-path I/O metrics.
|
||||
|
||||
`E-011` extension/runtime consumers:
|
||||
|
||||
- Need explicit ownership for runtime admission snapshots before extensions can
|
||||
observe scheduler or backpressure state.
|
||||
- Must not receive mutable handles to `ConcurrencyManager`, heal queues, or
|
||||
scanner budget tokens.
|
||||
|
||||
`C-011` controller work:
|
||||
|
||||
- Needs desired/current/status snapshots for request admission, scanner budget,
|
||||
and heal queue pressure before any controller can reconcile them.
|
||||
- Must keep worker mutation explicit. Read-only status should report `None` or
|
||||
no-op mutation until a reviewed worker lifecycle PR exists.
|
||||
|
||||
## Preservation Invariants
|
||||
|
||||
- Request reads must keep the same disk-read semaphore admission and active GET
|
||||
accounting.
|
||||
- I/O queue status and congestion metrics must remain derived from the same
|
||||
permit counts.
|
||||
- Scanner budget cancellation must keep its reason as runtime, objects, or
|
||||
directories.
|
||||
- Scanner inline-heal compatibility must continue to use asynchronous heal
|
||||
admission.
|
||||
- Heal duplicate admission must prefer merge semantics before full-queue
|
||||
rejection.
|
||||
- High-priority heal admission must still be able to displace lower-priority
|
||||
queued work where the current manager allows it.
|
||||
- Tokio runtime env names and fallback defaults must remain unchanged.
|
||||
@@ -4,7 +4,7 @@
|
||||
mod storage_compat;
|
||||
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
use crate::storage_compat::ecstore::bucket::utils::{
|
||||
use crate::storage_compat::{
|
||||
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
|
||||
};
|
||||
|
||||
|
||||
@@ -12,12 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod ecstore {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) mod utils {
|
||||
pub(crate) use rustfs_ecstore::bucket::utils::{
|
||||
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
pub(crate) use rustfs_ecstore::bucket::utils::{
|
||||
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
|
||||
};
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
mod storage_compat;
|
||||
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
use crate::storage_compat::ecstore::bucket::utils::{check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix};
|
||||
use crate::storage_compat::{check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix};
|
||||
use rustfs_utils::path::{clean, path_join};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
|
||||
@@ -12,12 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod ecstore {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) mod utils {
|
||||
pub(crate) use rustfs_ecstore::bucket::utils::{
|
||||
check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
pub(crate) use rustfs_ecstore::bucket::utils::{
|
||||
check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix,
|
||||
};
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::admin::auth::authenticate_request;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::storage_compat::ecstore::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::admin::storage_compat::versioning_sys::BucketVersioningSys;
|
||||
use crate::app::context::resolve_object_store_handle;
|
||||
use crate::auth::get_condition_values;
|
||||
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||
|
||||
@@ -24,7 +24,7 @@ pub(crate) async fn load_server_config_from_store() -> S3Result<Config> {
|
||||
return Ok(Config::new());
|
||||
};
|
||||
|
||||
crate::admin::storage_compat::ecstore::config::com::read_config_without_migrate(store)
|
||||
crate::admin::storage_compat::com::read_config_without_migrate(store)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))
|
||||
}
|
||||
@@ -82,7 +82,7 @@ where
|
||||
return Err(s3_error!(InternalError, "server storage not initialized"));
|
||||
};
|
||||
|
||||
let mut config = crate::admin::storage_compat::ecstore::config::com::read_config_without_migrate(store.clone())
|
||||
let mut config = crate::admin::storage_compat::com::read_config_without_migrate(store.clone())
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
|
||||
|
||||
@@ -90,7 +90,7 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
crate::admin::storage_compat::ecstore::config::com::save_server_config(store, &config)
|
||||
crate::admin::storage_compat::com::save_server_config(store, &config)
|
||||
.await
|
||||
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
|
||||
|
||||
|
||||
@@ -12,19 +12,17 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::admin::storage_compat::ecstore::bucket::utils::{deserialize, serialize};
|
||||
use crate::admin::storage_compat::ecstore::{
|
||||
bucket::{
|
||||
metadata::{
|
||||
BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
|
||||
BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG,
|
||||
BucketMetadata, OBJECT_LOCK_CONFIG,
|
||||
},
|
||||
metadata_sys,
|
||||
quota::BucketQuota,
|
||||
target::BucketTargets,
|
||||
use crate::admin::storage_compat::utils::{deserialize, serialize};
|
||||
use crate::admin::storage_compat::{
|
||||
StorageError,
|
||||
metadata::{
|
||||
BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
|
||||
BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG,
|
||||
BucketMetadata, OBJECT_LOCK_CONFIG,
|
||||
},
|
||||
error::StorageError,
|
||||
metadata_sys,
|
||||
quota::BucketQuota,
|
||||
target::BucketTargets,
|
||||
};
|
||||
use crate::{
|
||||
admin::{
|
||||
|
||||
@@ -18,12 +18,12 @@ use crate::admin::service::config::{
|
||||
apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem, signal_config_snapshot_reload, signal_dynamic_config_reload,
|
||||
validate_server_config,
|
||||
};
|
||||
use crate::admin::storage_compat::ecstore::config::com::STORAGE_CLASS_SUB_SYS;
|
||||
use crate::admin::storage_compat::ecstore::config::com::{
|
||||
use crate::admin::storage_compat::RUSTFS_META_BUCKET;
|
||||
use crate::admin::storage_compat::com::STORAGE_CLASS_SUB_SYS;
|
||||
use crate::admin::storage_compat::com::{
|
||||
delete_config, read_config, read_config_without_migrate, save_config, save_server_config,
|
||||
};
|
||||
use crate::admin::storage_compat::ecstore::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
|
||||
use crate::admin::storage_compat::ecstore::disk::RUSTFS_META_BUCKET;
|
||||
use crate::admin::storage_compat::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
|
||||
use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body};
|
||||
use crate::app::context::resolve_object_store_handle;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
@@ -711,7 +711,7 @@ fn success_response(config_applied: bool) -> S3Result<S3Response<(StatusCode, Bo
|
||||
Ok(S3Response::with_headers((StatusCode::OK, Body::default()), headers))
|
||||
}
|
||||
|
||||
fn object_store() -> S3Result<std::sync::Arc<crate::admin::storage_compat::ecstore::store::ECStore>> {
|
||||
fn object_store() -> S3Result<std::sync::Arc<crate::admin::storage_compat::ECStore>> {
|
||||
resolve_object_store_handle().ok_or_else(|| s3_error!(InternalError, "server storage not initialized"))
|
||||
}
|
||||
|
||||
@@ -756,7 +756,7 @@ fn config_update_sub_system(directives: &[ConfigDirective]) -> S3Result<Option<&
|
||||
|
||||
fn validate_config_directives(directives: &[ConfigDirective]) -> S3Result<()> {
|
||||
if DEFAULT_KVS.get().is_none() {
|
||||
crate::admin::storage_compat::ecstore::config::init();
|
||||
crate::admin::storage_compat::init();
|
||||
}
|
||||
let Some(defaults) = DEFAULT_KVS.get() else {
|
||||
return Err(s3_error!(InternalError, "config defaults are not initialized"));
|
||||
@@ -1410,7 +1410,7 @@ fn env_help_key(sub_system: &str, key: &str) -> String {
|
||||
|
||||
fn default_help_postfix(sub_system: &str, key: &str) -> String {
|
||||
if DEFAULT_KVS.get().is_none() {
|
||||
crate::admin::storage_compat::ecstore::config::init();
|
||||
crate::admin::storage_compat::init();
|
||||
}
|
||||
|
||||
DEFAULT_KVS
|
||||
@@ -1897,7 +1897,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn full_config_export_can_be_reapplied() {
|
||||
crate::admin::storage_compat::ecstore::config::init();
|
||||
crate::admin::storage_compat::init();
|
||||
let mut original = ServerConfig::new();
|
||||
apply_set_directives(
|
||||
&mut original,
|
||||
@@ -1944,7 +1944,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
|
||||
|
||||
#[test]
|
||||
fn build_help_response_appends_default_value_postfix() {
|
||||
crate::admin::storage_compat::ecstore::config::init();
|
||||
crate::admin::storage_compat::init();
|
||||
let response = build_help_response(Some("identity_openid"), Some("scopes"), false).expect("help response");
|
||||
|
||||
assert_eq!(response.keys_help.len(), 2);
|
||||
@@ -2057,7 +2057,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
|
||||
|
||||
#[test]
|
||||
fn render_selected_config_includes_env_override_lines() {
|
||||
crate::admin::storage_compat::ecstore::config::init();
|
||||
crate::admin::storage_compat::init();
|
||||
temp_env::with_vars(
|
||||
[
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example")),
|
||||
@@ -2093,7 +2093,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
|
||||
|
||||
#[test]
|
||||
fn render_selected_config_lists_env_only_targets() {
|
||||
crate::admin::storage_compat::ecstore::config::init();
|
||||
crate::admin::storage_compat::init();
|
||||
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || {
|
||||
let config = ServerConfig::new();
|
||||
let rendered = String::from_utf8(
|
||||
@@ -2116,7 +2116,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
|
||||
|
||||
#[test]
|
||||
fn render_selected_config_supports_specific_env_only_target_queries() {
|
||||
crate::admin::storage_compat::ecstore::config::init();
|
||||
crate::admin::storage_compat::init();
|
||||
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || {
|
||||
let config = ServerConfig::new();
|
||||
let rendered = String::from_utf8(
|
||||
@@ -2139,7 +2139,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
|
||||
|
||||
#[test]
|
||||
fn render_selected_config_orders_default_before_named_targets() {
|
||||
crate::admin::storage_compat::ecstore::config::init();
|
||||
crate::admin::storage_compat::init();
|
||||
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ALPHA", Some("http://alpha.example"))], || {
|
||||
let mut config = ServerConfig::new();
|
||||
apply_set_directives(
|
||||
@@ -2320,7 +2320,7 @@ identity_openid client_id="existing-client""#,
|
||||
|
||||
#[test]
|
||||
fn storage_class_get_target_none_matches_full_export() {
|
||||
crate::admin::storage_compat::ecstore::config::init();
|
||||
crate::admin::storage_compat::init();
|
||||
let mut config = ServerConfig::new();
|
||||
apply_set_directives(
|
||||
&mut config,
|
||||
|
||||
@@ -14,8 +14,8 @@
|
||||
|
||||
use crate::admin::auth::{authenticate_request, validate_admin_request};
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::storage_compat::ecstore::bucket::utils::is_valid_object_prefix;
|
||||
use crate::admin::storage_compat::ecstore::store_utils::is_reserved_or_invalid_bucket;
|
||||
use crate::admin::storage_compat::is_reserved_or_invalid_bucket;
|
||||
use crate::admin::storage_compat::utils::is_valid_object_prefix;
|
||||
use crate::app::context::resolve_object_store_handle;
|
||||
use crate::server::ADMIN_PREFIX;
|
||||
use crate::server::RemoteAddr;
|
||||
@@ -345,10 +345,10 @@ fn should_handle_root_heal_directly(hip: &HealInitParams) -> bool {
|
||||
hip.bucket.is_empty() && hip.obj_prefix.is_empty() && hip.client_token.is_empty() && !hip.force_stop
|
||||
}
|
||||
|
||||
fn map_root_heal_status(heal_err: Option<crate::admin::storage_compat::ecstore::error::Error>) -> S3Result<()> {
|
||||
fn map_root_heal_status(heal_err: Option<crate::admin::storage_compat::Error>) -> S3Result<()> {
|
||||
match heal_err {
|
||||
None => Ok(()),
|
||||
Some(crate::admin::storage_compat::ecstore::error::StorageError::NoHealRequired) => {
|
||||
Some(crate::admin::storage_compat::StorageError::NoHealRequired) => {
|
||||
info!(
|
||||
event = EVENT_ADMIN_RESPONSE_EMITTED,
|
||||
component = LOG_COMPONENT_ADMIN_API,
|
||||
@@ -703,7 +703,7 @@ mod tests {
|
||||
encode_heal_task_status, heal_channel_response_items, heal_channel_response_summary, json_response, map_heal_response,
|
||||
map_root_heal_status, should_handle_root_heal_directly, validate_heal_request_mode, validate_heal_target,
|
||||
};
|
||||
use crate::admin::storage_compat::ecstore::error::StorageError;
|
||||
use crate::admin::storage_compat::StorageError;
|
||||
use bytes::Bytes;
|
||||
use http::StatusCode;
|
||||
use http::Uri;
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
|
||||
use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::storage_compat::ecstore::config::com::{read_config, save_config};
|
||||
use crate::admin::storage_compat::com::{read_config, save_config};
|
||||
use crate::app::context::{resolve_kms_runtime_service_manager, resolve_object_store_handle};
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
|
||||
use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::router::Operation;
|
||||
use crate::admin::storage_compat::ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
||||
use crate::admin::storage_compat::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::server::RemoteAddr;
|
||||
use crate::storage::request_context::spawn_traced;
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, Operation, S3Router};
|
||||
use crate::admin::storage_compat::ecstore::global::get_global_region;
|
||||
use crate::admin::storage_compat::get_global_region;
|
||||
use crate::app::context::resolve_object_store_handle;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::error::ApiError;
|
||||
@@ -649,7 +649,7 @@ async fn preflight_zip_items(request: &CreateObjectZipDownloadRequest, items: &[
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn storage_error_to_s3(err: crate::admin::storage_compat::ecstore::error::Error) -> s3s::S3Error {
|
||||
fn storage_error_to_s3(err: crate::admin::storage_compat::Error) -> s3s::S3Error {
|
||||
ApiError::from(err).into()
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use super::sts::create_oidc_sts_credentials;
|
||||
use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::storage_compat::ecstore::config::com::{read_config_without_migrate, save_server_config};
|
||||
use crate::admin::storage_compat::com::{read_config_without_migrate, save_server_config};
|
||||
use crate::app::context::resolve_object_store_handle;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::server::{ADMIN_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr};
|
||||
|
||||
@@ -99,7 +99,7 @@ macro_rules! log_pool_response_emitted {
|
||||
};
|
||||
}
|
||||
|
||||
fn endpoints_from_context() -> Option<crate::admin::storage_compat::ecstore::endpoints::EndpointServerPools> {
|
||||
fn endpoints_from_context() -> Option<crate::admin::storage_compat::EndpointServerPools> {
|
||||
resolve_endpoints_handle()
|
||||
}
|
||||
|
||||
|
||||
@@ -16,9 +16,9 @@
|
||||
|
||||
use crate::admin::auth::{validate_admin_request, validate_admin_request_with_bucket};
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::storage_compat::ecstore::bucket::metadata_sys::BucketMetadataSys;
|
||||
use crate::admin::storage_compat::ecstore::bucket::quota::checker::QuotaChecker;
|
||||
use crate::admin::storage_compat::ecstore::bucket::quota::{BucketQuota, QuotaError, QuotaOperation};
|
||||
use crate::admin::storage_compat::metadata_sys::BucketMetadataSys;
|
||||
use crate::admin::storage_compat::quota::checker::QuotaChecker;
|
||||
use crate::admin::storage_compat::quota::{BucketQuota, QuotaError, QuotaOperation};
|
||||
use crate::app::context::{resolve_bucket_metadata_handle, resolve_object_store_handle};
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::server::ADMIN_PREFIX;
|
||||
@@ -175,7 +175,7 @@ async fn current_usage_from_context(bucket: &str) -> u64 {
|
||||
return 0;
|
||||
};
|
||||
|
||||
match crate::admin::storage_compat::ecstore::data_usage::load_data_usage_from_backend(store).await {
|
||||
match crate::admin::storage_compat::load_data_usage_from_backend(store).await {
|
||||
Ok(data_usage_info) => data_usage_info
|
||||
.buckets_usage
|
||||
.get(bucket)
|
||||
|
||||
@@ -12,10 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::admin::storage_compat::ecstore::{
|
||||
error::StorageError,
|
||||
notification_sys::get_global_notification_sys,
|
||||
rebalance::{DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta},
|
||||
use crate::admin::storage_compat::{
|
||||
DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, StorageError, get_global_notification_sys,
|
||||
};
|
||||
use crate::{
|
||||
admin::{
|
||||
@@ -150,7 +148,7 @@ fn build_rebalance_pool_progress(
|
||||
now: OffsetDateTime,
|
||||
stop_time: Option<OffsetDateTime>,
|
||||
percent_free_goal: f64,
|
||||
ps: &crate::admin::storage_compat::ecstore::rebalance::RebalanceStats,
|
||||
ps: &crate::admin::storage_compat::RebalanceStats,
|
||||
) -> Option<RebalPoolProgress> {
|
||||
let total_bytes_to_rebal = ps.init_capacity as f64 * percent_free_goal - ps.init_free_space as f64;
|
||||
let terminal_time = ps.info.end_time.or(stop_time);
|
||||
@@ -193,7 +191,7 @@ fn build_rebalance_pool_statuses(
|
||||
now: OffsetDateTime,
|
||||
stop_time: Option<OffsetDateTime>,
|
||||
percent_free_goal: f64,
|
||||
pool_stats: &[crate::admin::storage_compat::ecstore::rebalance::RebalanceStats],
|
||||
pool_stats: &[crate::admin::storage_compat::RebalanceStats],
|
||||
disk_stats: &[DiskStat],
|
||||
) -> Vec<RebalancePoolStatus> {
|
||||
pool_stats
|
||||
@@ -563,9 +561,7 @@ mod rebalance_handler_tests {
|
||||
RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, build_rebalance_pool_statuses, rebalance_pool_used,
|
||||
rebalance_remaining_buckets, rebalance_used_pct,
|
||||
};
|
||||
use crate::admin::storage_compat::ecstore::rebalance::{
|
||||
DiskStat, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceStats,
|
||||
};
|
||||
use crate::admin::storage_compat::{DiskStat, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceStats};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -15,15 +15,15 @@
|
||||
use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::handlers::site_replication::site_replication_peer_deployment_id_for_endpoint;
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::{BucketTargetError, BucketTargetSys};
|
||||
use crate::admin::storage_compat::ecstore::bucket::metadata::BUCKET_TARGETS_FILE;
|
||||
use crate::admin::storage_compat::ecstore::bucket::metadata_sys;
|
||||
use crate::admin::storage_compat::ecstore::bucket::metadata_sys::get_replication_config;
|
||||
use crate::admin::storage_compat::ecstore::bucket::replication::BucketStats;
|
||||
use crate::admin::storage_compat::ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
|
||||
use crate::admin::storage_compat::ecstore::bucket::target::BucketTarget;
|
||||
use crate::admin::storage_compat::ecstore::error::StorageError;
|
||||
use crate::admin::storage_compat::ecstore::global::global_rustfs_port;
|
||||
use crate::admin::storage_compat::StorageError;
|
||||
use crate::admin::storage_compat::bucket_target_sys::{BucketTargetError, BucketTargetSys};
|
||||
use crate::admin::storage_compat::global_rustfs_port;
|
||||
use crate::admin::storage_compat::metadata::BUCKET_TARGETS_FILE;
|
||||
use crate::admin::storage_compat::metadata_sys;
|
||||
use crate::admin::storage_compat::metadata_sys::get_replication_config;
|
||||
use crate::admin::storage_compat::replication::BucketStats;
|
||||
use crate::admin::storage_compat::replication::GLOBAL_REPLICATION_STATS;
|
||||
use crate::admin::storage_compat::target::BucketTarget;
|
||||
use crate::admin::utils::read_compatible_admin_body;
|
||||
use crate::app::context::resolve_object_store_handle;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
|
||||
@@ -18,24 +18,20 @@ use crate::admin::site_replication_identity::{
|
||||
canonical_endpoint, deployment_id_for_endpoint, normalize_peer_map_by_identity_with, same_identity_endpoint,
|
||||
site_identity_key,
|
||||
};
|
||||
use crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::BucketTargetSys;
|
||||
use crate::admin::storage_compat::ecstore::bucket::metadata::{
|
||||
use crate::admin::storage_compat::Error as StorageError;
|
||||
use crate::admin::storage_compat::bucket_target_sys::BucketTargetSys;
|
||||
use crate::admin::storage_compat::com::{delete_config, read_config, save_config};
|
||||
use crate::admin::storage_compat::metadata::{
|
||||
BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG,
|
||||
BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG,
|
||||
};
|
||||
use crate::admin::storage_compat::ecstore::bucket::metadata_sys;
|
||||
use crate::admin::storage_compat::ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
|
||||
use crate::admin::storage_compat::ecstore::bucket::replication::{
|
||||
ReplicationConfigurationExt, ResyncOpts, get_global_replication_pool,
|
||||
};
|
||||
use crate::admin::storage_compat::ecstore::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials};
|
||||
use crate::admin::storage_compat::ecstore::bucket::utils::{deserialize, serialize};
|
||||
use crate::admin::storage_compat::ecstore::bucket::versioning::VersioningApi;
|
||||
use crate::admin::storage_compat::ecstore::config::com::{delete_config, read_config, save_config};
|
||||
use crate::admin::storage_compat::ecstore::error::Error as StorageError;
|
||||
use crate::admin::storage_compat::ecstore::global::{
|
||||
get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port,
|
||||
};
|
||||
use crate::admin::storage_compat::metadata_sys;
|
||||
use crate::admin::storage_compat::replication::GLOBAL_REPLICATION_STATS;
|
||||
use crate::admin::storage_compat::replication::{ReplicationConfigurationExt, ResyncOpts, get_global_replication_pool};
|
||||
use crate::admin::storage_compat::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials};
|
||||
use crate::admin::storage_compat::utils::{deserialize, serialize};
|
||||
use crate::admin::storage_compat::versioning::VersioningApi;
|
||||
use crate::admin::storage_compat::{get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port};
|
||||
use crate::admin::utils::{encode_compatible_admin_payload, read_compatible_admin_body};
|
||||
use crate::app::context::resolve_object_store_handle;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
@@ -656,7 +652,7 @@ async fn site_replication_peer_client() -> S3Result<reqwest::Client> {
|
||||
built
|
||||
}
|
||||
|
||||
fn runtime_tls_enabled_with(endpoints: Option<&crate::admin::storage_compat::ecstore::endpoints::EndpointServerPools>) -> bool {
|
||||
fn runtime_tls_enabled_with(endpoints: Option<&crate::admin::storage_compat::EndpointServerPools>) -> bool {
|
||||
if !rustfs_utils::get_env_str(ENV_RUSTFS_TLS_PATH, DEFAULT_RUSTFS_TLS_PATH).is_empty() {
|
||||
return true;
|
||||
}
|
||||
@@ -3357,7 +3353,7 @@ fn is_stale_update(local_updated_at: OffsetDateTime, incoming_updated_at: Option
|
||||
}
|
||||
|
||||
fn bucket_meta_local_updated_at(
|
||||
bucket_meta: &crate::admin::storage_compat::ecstore::bucket::metadata::BucketMetadata,
|
||||
bucket_meta: &crate::admin::storage_compat::metadata::BucketMetadata,
|
||||
config_file: &str,
|
||||
) -> OffsetDateTime {
|
||||
match config_file {
|
||||
@@ -4578,8 +4574,8 @@ impl Operation for SRRotateServiceAccountHandler {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::admin::storage_compat::ecstore::disk::endpoint::Endpoint;
|
||||
use crate::admin::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::admin::storage_compat::Endpoint;
|
||||
use crate::admin::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use http::{HeaderMap, HeaderValue, Uri};
|
||||
use rustfs_common::{get_global_outbound_tls_generation, set_global_outbound_tls_generation};
|
||||
use rustfs_policy::policy::action::S3Action;
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::is_admin::IsAdminHandler;
|
||||
use crate::admin::storage_compat::ecstore::bucket::utils::serialize;
|
||||
use crate::admin::storage_compat::utils::serialize;
|
||||
use crate::{
|
||||
admin::{
|
||||
handlers::site_replication::site_replication_iam_change_hook,
|
||||
|
||||
@@ -12,10 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::admin::storage_compat::ecstore::{
|
||||
bucket::{metadata::table_catalog_path_hash, metadata_sys},
|
||||
store::ECStore,
|
||||
};
|
||||
use crate::admin::storage_compat::{ECStore, metadata::table_catalog_path_hash, metadata_sys};
|
||||
use crate::admin::{
|
||||
auth::{AdminResourceScope, validate_admin_request, validate_admin_request_with_bucket_object},
|
||||
router::{AdminOperation, Operation, S3Router},
|
||||
|
||||
@@ -13,21 +13,11 @@
|
||||
// limitations under the License.
|
||||
#![allow(unused_variables, unused_mut, unused_must_use)]
|
||||
|
||||
use crate::admin::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState;
|
||||
use crate::admin::storage_compat::ecstore::{
|
||||
bucket::lifecycle::tier_last_day_stats::DailyAllTierStats,
|
||||
client::admin_handler_utils::AdminError,
|
||||
config::storageclass,
|
||||
notification_sys::get_global_notification_sys,
|
||||
tier::{
|
||||
tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_MISSING_CREDENTIALS},
|
||||
tier_admin::TierCreds,
|
||||
tier_config::{TierConfig, TierType},
|
||||
tier_handlers::{
|
||||
ERR_TIER_ALREADY_EXISTS, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE,
|
||||
ERR_TIER_NOT_FOUND,
|
||||
},
|
||||
},
|
||||
use crate::admin::storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState;
|
||||
use crate::admin::storage_compat::{
|
||||
AdminError, DailyAllTierStats, ERR_TIER_ALREADY_EXISTS, ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY,
|
||||
ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE,
|
||||
ERR_TIER_NOT_FOUND, TierConfig, TierCreds, TierType, get_global_notification_sys, storageclass,
|
||||
};
|
||||
use crate::{
|
||||
admin::{
|
||||
@@ -916,7 +906,7 @@ impl Operation for ClearTier {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::admin::storage_compat::ecstore::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
|
||||
use crate::admin::storage_compat::lifecycle::tier_last_day_stats::LastDayTierStats;
|
||||
use http::Uri;
|
||||
use matchit::Router;
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::admin::router::Operation;
|
||||
use crate::admin::storage_compat::ecstore::rpc::PeerRestClient;
|
||||
use crate::admin::storage_compat::PeerRestClient;
|
||||
use crate::app::context::resolve_endpoints_handle;
|
||||
use http::StatusCode;
|
||||
use hyper::Uri;
|
||||
|
||||
+25
-27
@@ -14,24 +14,24 @@
|
||||
|
||||
use crate::admin::console::{is_console_path, make_console_server};
|
||||
use crate::admin::handlers::oidc::is_oidc_path;
|
||||
use crate::admin::storage_compat::ecstore::bucket::bandwidth::monitor::BandwidthDetails;
|
||||
use crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::{
|
||||
use crate::admin::storage_compat::GLOBAL_BOOT_TIME;
|
||||
use crate::admin::storage_compat::PeerRestClient;
|
||||
use crate::admin::storage_compat::bandwidth::monitor::BandwidthDetails;
|
||||
use crate::admin::storage_compat::bucket_target_sys::{
|
||||
BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, TargetClient,
|
||||
};
|
||||
use crate::admin::storage_compat::ecstore::bucket::metadata::BUCKET_TARGETS_FILE;
|
||||
use crate::admin::storage_compat::ecstore::bucket::metadata_sys;
|
||||
use crate::admin::storage_compat::ecstore::bucket::replication::{
|
||||
use crate::admin::storage_compat::com::read_config_without_migrate;
|
||||
use crate::admin::storage_compat::get_global_notification_sys;
|
||||
use crate::admin::storage_compat::metadata::BUCKET_TARGETS_FILE;
|
||||
use crate::admin::storage_compat::metadata_sys;
|
||||
use crate::admin::storage_compat::replication::{
|
||||
BucketReplicationResyncStatus, BucketStats, GLOBAL_REPLICATION_STATS, ObjectOpts, ReplicationConfigurationExt, ResyncOpts,
|
||||
get_global_replication_pool,
|
||||
};
|
||||
use crate::admin::storage_compat::ecstore::bucket::target::{BucketTarget, BucketTargetType, BucketTargets};
|
||||
use crate::admin::storage_compat::ecstore::bucket::versioning::VersioningApi;
|
||||
use crate::admin::storage_compat::ecstore::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::admin::storage_compat::ecstore::config::com::read_config_without_migrate;
|
||||
use crate::admin::storage_compat::ecstore::global::GLOBAL_BOOT_TIME;
|
||||
use crate::admin::storage_compat::ecstore::global::{get_global_bucket_monitor, get_global_deployment_id, get_global_region};
|
||||
use crate::admin::storage_compat::ecstore::notification_sys::get_global_notification_sys;
|
||||
use crate::admin::storage_compat::ecstore::rpc::PeerRestClient;
|
||||
use crate::admin::storage_compat::target::{BucketTarget, BucketTargetType, BucketTargets};
|
||||
use crate::admin::storage_compat::versioning::VersioningApi;
|
||||
use crate::admin::storage_compat::versioning_sys::BucketVersioningSys;
|
||||
use crate::admin::storage_compat::{get_global_bucket_monitor, get_global_deployment_id, get_global_region};
|
||||
use crate::app::context::resolve_object_store_handle;
|
||||
use crate::app::object_usecase::DefaultObjectUsecase;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
@@ -1420,9 +1420,7 @@ async fn ensure_replication_bucket_exists(bucket: &str) -> S3Result<()> {
|
||||
async fn ensure_replication_config_exists(bucket: &str) -> S3Result<()> {
|
||||
match metadata_sys::get_replication_config(bucket).await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(crate::admin::storage_compat::ecstore::error::StorageError::ConfigNotFound) => {
|
||||
Err(s3_error!(ReplicationConfigurationNotFoundError))
|
||||
}
|
||||
Err(crate::admin::storage_compat::StorageError::ConfigNotFound) => Err(s3_error!(ReplicationConfigurationNotFoundError)),
|
||||
Err(err) => Err(ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
@@ -1947,7 +1945,7 @@ async fn resolve_replication_target_client(bucket: &str, target: &BucketTarget)
|
||||
|
||||
fn build_replication_probe_put_options(now: OffsetDateTime) -> PutObjectOptions {
|
||||
PutObjectOptions {
|
||||
internal: crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::AdvancedPutOptions {
|
||||
internal: crate::admin::storage_compat::bucket_target_sys::AdvancedPutOptions {
|
||||
source_version_id: Uuid::new_v4().to_string(),
|
||||
replication_status: ReplicationStatusType::Replica,
|
||||
source_mtime: now,
|
||||
@@ -2062,7 +2060,7 @@ async fn source_bucket_requires_object_lock(bucket: &str) -> S3Result<bool> {
|
||||
.object_lock_enabled
|
||||
.as_ref()
|
||||
.is_some_and(|state| state.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)),
|
||||
Err(crate::admin::storage_compat::ecstore::error::StorageError::ConfigNotFound) => Ok(false),
|
||||
Err(crate::admin::storage_compat::StorageError::ConfigNotFound) => Ok(false),
|
||||
Err(err) => Err(ApiError::from(err).into()),
|
||||
}
|
||||
}
|
||||
@@ -2776,7 +2774,7 @@ mod tests {
|
||||
#[test]
|
||||
fn apply_replication_reset_to_targets_updates_matching_target() {
|
||||
let mut targets = BucketTargets {
|
||||
targets: vec![crate::admin::storage_compat::ecstore::bucket::target::BucketTarget {
|
||||
targets: vec![crate::admin::storage_compat::target::BucketTarget {
|
||||
arn: "arn:target".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
@@ -2798,10 +2796,10 @@ mod tests {
|
||||
let mut status = BucketReplicationResyncStatus::new();
|
||||
status.targets_map.insert(
|
||||
"arn:z".to_string(),
|
||||
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus {
|
||||
crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
|
||||
resync_id: "rid-z".to_string(),
|
||||
last_update: Some(datetime!(2025-01-03 00:00 UTC)),
|
||||
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncFailed,
|
||||
resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncFailed,
|
||||
failed_count: 2,
|
||||
failed_size: 4,
|
||||
bucket: "bucket-z".to_string(),
|
||||
@@ -2811,10 +2809,10 @@ mod tests {
|
||||
);
|
||||
status.targets_map.insert(
|
||||
"arn:a".to_string(),
|
||||
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus {
|
||||
crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
|
||||
resync_id: "rid-a".to_string(),
|
||||
last_update: Some(datetime!(2025-01-02 00:00 UTC)),
|
||||
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncCompleted,
|
||||
resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncCompleted,
|
||||
replicated_count: 3,
|
||||
replicated_size: 9,
|
||||
bucket: "bucket-a".to_string(),
|
||||
@@ -2844,10 +2842,10 @@ mod tests {
|
||||
let mut status = BucketReplicationResyncStatus::new();
|
||||
status.targets_map.insert(
|
||||
"arn:z".to_string(),
|
||||
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus {
|
||||
crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
|
||||
resync_id: "rid-z".to_string(),
|
||||
last_update: Some(datetime!(2025-02-03 00:00 UTC)),
|
||||
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncFailed,
|
||||
resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncFailed,
|
||||
failed_count: 2,
|
||||
failed_size: 4,
|
||||
bucket: "bucket-z".to_string(),
|
||||
@@ -2857,10 +2855,10 @@ mod tests {
|
||||
);
|
||||
status.targets_map.insert(
|
||||
"arn:a".to_string(),
|
||||
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus {
|
||||
crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
|
||||
resync_id: "rid-a".to_string(),
|
||||
last_update: Some(datetime!(2025-02-02 00:00 UTC)),
|
||||
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncCompleted,
|
||||
resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncCompleted,
|
||||
replicated_count: 3,
|
||||
replicated_size: 9,
|
||||
bucket: "bucket-a".to_string(),
|
||||
|
||||
@@ -12,10 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::admin::storage_compat::ecstore::config::com::{STORAGE_CLASS_SUB_SYS, read_config_without_migrate};
|
||||
use crate::admin::storage_compat::ecstore::config::set_global_storage_class;
|
||||
use crate::admin::storage_compat::ecstore::config::storageclass;
|
||||
use crate::admin::storage_compat::ecstore::notification_sys::get_global_notification_sys;
|
||||
use crate::admin::storage_compat::com::{STORAGE_CLASS_SUB_SYS, read_config_without_migrate};
|
||||
use crate::admin::storage_compat::get_global_notification_sys;
|
||||
use crate::admin::storage_compat::set_global_storage_class;
|
||||
use crate::admin::storage_compat::storageclass;
|
||||
use crate::app::context::resolve_object_store_handle;
|
||||
use rustfs_audit::reload_audit_config;
|
||||
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_WEBHOOK_SUB_SYS};
|
||||
@@ -371,7 +371,7 @@ pub async fn signal_config_snapshot_reload() {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::admin::storage_compat::ecstore::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG};
|
||||
use crate::admin::storage_compat::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG};
|
||||
use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS;
|
||||
use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES};
|
||||
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
|
||||
@@ -427,7 +427,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn validate_notify_subsystem_config_rejects_invalid_webhook_endpoint() {
|
||||
crate::admin::storage_compat::ecstore::config::init();
|
||||
crate::admin::storage_compat::init();
|
||||
let mut config = ServerConfig::new();
|
||||
let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults");
|
||||
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
|
||||
@@ -441,7 +441,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn validate_audit_subsystem_config_rejects_relative_queue_dir() {
|
||||
crate::admin::storage_compat::ecstore::config::init();
|
||||
crate::admin::storage_compat::init();
|
||||
let mut config = ServerConfig::new();
|
||||
let targets = config.0.get_mut(AUDIT_MQTT_SUB_SYS).expect("audit mqtt defaults");
|
||||
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
|
||||
@@ -456,7 +456,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn validate_identity_openid_config_rejects_missing_openid_scope() {
|
||||
crate::admin::storage_compat::ecstore::config::init();
|
||||
crate::admin::storage_compat::init();
|
||||
let mut config = ServerConfig::new();
|
||||
let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults");
|
||||
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
|
||||
@@ -473,7 +473,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn validate_identity_openid_config_rejects_invalid_named_provider_id() {
|
||||
crate::admin::storage_compat::ecstore::config::init();
|
||||
crate::admin::storage_compat::init();
|
||||
let mut config = ServerConfig::new();
|
||||
let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults");
|
||||
let default_kvs = targets.get(DEFAULT_DELIMITER).cloned().expect("default target");
|
||||
|
||||
@@ -13,8 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::admin::site_replication_identity::{deployment_id_for_endpoint, normalize_peer_map_by_identity_with};
|
||||
use crate::admin::storage_compat::ecstore::config::com::{read_config, save_config};
|
||||
use crate::admin::storage_compat::ecstore::error::Error as StorageError;
|
||||
use crate::admin::storage_compat::Error as StorageError;
|
||||
use crate::admin::storage_compat::com::{read_config, save_config};
|
||||
use crate::app::context::resolve_object_store_handle;
|
||||
use rustfs_madmin::PeerInfo;
|
||||
use s3s::{S3Error, S3ErrorCode, S3Result};
|
||||
|
||||
@@ -12,78 +12,36 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod ecstore {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use rustfs_ecstore::bucket::{
|
||||
bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning,
|
||||
versioning_sys,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod client {
|
||||
pub(crate) use rustfs_ecstore::client::admin_handler_utils;
|
||||
}
|
||||
|
||||
pub(crate) mod config {
|
||||
pub(crate) use rustfs_ecstore::config::{com, init, set_global_storage_class, storageclass};
|
||||
}
|
||||
|
||||
pub(crate) mod data_usage {
|
||||
pub(crate) use rustfs_ecstore::data_usage::load_data_usage_from_backend;
|
||||
}
|
||||
|
||||
pub(crate) mod disk {
|
||||
pub(crate) use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::disk::endpoint;
|
||||
}
|
||||
|
||||
pub(crate) mod endpoints {
|
||||
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
|
||||
}
|
||||
|
||||
pub(crate) mod error {
|
||||
pub(crate) use rustfs_ecstore::error::{Error, StorageError};
|
||||
}
|
||||
|
||||
pub(crate) mod global {
|
||||
pub(crate) use rustfs_ecstore::global::{
|
||||
GLOBAL_BOOT_TIME, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt, get_global_region,
|
||||
global_rustfs_port,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod metrics_realtime {
|
||||
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
||||
}
|
||||
|
||||
pub(crate) mod notification_sys {
|
||||
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
|
||||
}
|
||||
|
||||
pub(crate) mod rebalance {
|
||||
pub(crate) use rustfs_ecstore::rebalance::{
|
||||
DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::rebalance::{RebalStatus, RebalanceInfo};
|
||||
}
|
||||
|
||||
pub(crate) mod rpc {
|
||||
pub(crate) use rustfs_ecstore::rpc::PeerRestClient;
|
||||
}
|
||||
|
||||
pub(crate) mod store {
|
||||
pub(crate) use rustfs_ecstore::store::ECStore;
|
||||
}
|
||||
|
||||
pub(crate) mod store_utils {
|
||||
pub(crate) use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket;
|
||||
}
|
||||
|
||||
pub(crate) mod tier {
|
||||
pub(crate) use rustfs_ecstore::tier::{tier, tier_admin, tier_config, tier_handlers};
|
||||
}
|
||||
}
|
||||
pub(crate) use rustfs_ecstore::bucket::lifecycle::tier_last_day_stats::DailyAllTierStats;
|
||||
pub(crate) use rustfs_ecstore::bucket::{
|
||||
bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning,
|
||||
versioning_sys,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::client::admin_handler_utils::AdminError;
|
||||
pub(crate) use rustfs_ecstore::config::{com, init, set_global_storage_class, storageclass};
|
||||
pub(crate) use rustfs_ecstore::data_usage::load_data_usage_from_backend;
|
||||
pub(crate) use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
|
||||
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
|
||||
pub(crate) use rustfs_ecstore::error::{Error, StorageError};
|
||||
pub(crate) use rustfs_ecstore::global::{
|
||||
GLOBAL_BOOT_TIME, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt, get_global_region,
|
||||
global_rustfs_port,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
||||
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
|
||||
pub(crate) use rustfs_ecstore::rebalance::{DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::rebalance::{RebalStatus, RebalanceInfo};
|
||||
pub(crate) use rustfs_ecstore::rpc::PeerRestClient;
|
||||
pub(crate) use rustfs_ecstore::store::ECStore;
|
||||
pub(crate) use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket;
|
||||
pub(crate) use rustfs_ecstore::tier::tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_MISSING_CREDENTIALS};
|
||||
pub(crate) use rustfs_ecstore::tier::tier_admin::TierCreds;
|
||||
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierType};
|
||||
pub(crate) use rustfs_ecstore::tier::tier_handlers::{
|
||||
ERR_TIER_ALREADY_EXISTS, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND,
|
||||
};
|
||||
|
||||
@@ -15,13 +15,11 @@
|
||||
//! Admin application use-case contracts.
|
||||
|
||||
use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
|
||||
use crate::app::storage_compat::ecstore::admin_server_info::get_server_info;
|
||||
use crate::app::storage_compat::ecstore::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend};
|
||||
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools;
|
||||
use crate::app::storage_compat::ecstore::pools::{
|
||||
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
||||
};
|
||||
use crate::app::storage_compat::ecstore::store::ECStore;
|
||||
use crate::app::storage_compat::ECStore;
|
||||
use crate::app::storage_compat::EndpointServerPools;
|
||||
use crate::app::storage_compat::get_server_info;
|
||||
use crate::app::storage_compat::{PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free};
|
||||
use crate::app::storage_compat::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend};
|
||||
use crate::capacity::resolve_admin_used_capacity;
|
||||
use crate::error::ApiError;
|
||||
use crate::server::{DependencyReadiness, collect_dependency_readiness as collect_runtime_dependency_readiness};
|
||||
@@ -318,7 +316,7 @@ impl DefaultAdminUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::storage_compat::ecstore::pools::{PoolDecommissionInfo, PoolStatus};
|
||||
use crate::app::storage_compat::{PoolDecommissionInfo, PoolStatus};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -20,7 +20,11 @@ use crate::admin::handlers::site_replication::{
|
||||
use crate::app::context::{
|
||||
AppContext, default_notify_interface, get_global_app_context, resolve_object_store_handle_for_context,
|
||||
};
|
||||
use crate::app::storage_compat::ecstore::bucket::{
|
||||
use crate::app::storage_compat::ECStore;
|
||||
use crate::app::storage_compat::StorageError;
|
||||
use crate::app::storage_compat::get_global_notification_sys;
|
||||
use crate::app::storage_compat::object_api_utils::to_s3s_etag;
|
||||
use crate::app::storage_compat::{
|
||||
bucket_target_sys::BucketTargetSys,
|
||||
lifecycle::bucket_lifecycle_ops::{
|
||||
enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, validate_transition_tier,
|
||||
@@ -38,10 +42,6 @@ use crate::app::storage_compat::ecstore::bucket::{
|
||||
versioning::VersioningApi,
|
||||
versioning_sys::BucketVersioningSys,
|
||||
};
|
||||
use crate::app::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
|
||||
use crate::app::storage_compat::ecstore::error::StorageError;
|
||||
use crate::app::storage_compat::ecstore::notification_sys::get_global_notification_sys;
|
||||
use crate::app::storage_compat::ecstore::store::ECStore;
|
||||
use crate::auth::get_condition_values_with_client_info;
|
||||
use crate::error::ApiError;
|
||||
use crate::server::RemoteAddr;
|
||||
@@ -1628,9 +1628,7 @@ impl DefaultBucketUsecase {
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(err) =
|
||||
crate::app::storage_compat::ecstore::bucket::lifecycle::lifecycle::Lifecycle::validate(&input_cfg, &rcfg).await
|
||||
{
|
||||
if let Err(err) = crate::app::storage_compat::lifecycle::lifecycle::Lifecycle::validate(&input_cfg, &rcfg).await {
|
||||
return Err(s3_error!(InvalidArgument, "{err}"));
|
||||
}
|
||||
|
||||
@@ -2280,7 +2278,7 @@ mod tests {
|
||||
BucketTargets {
|
||||
targets: arns
|
||||
.iter()
|
||||
.map(|arn| crate::app::storage_compat::ecstore::bucket::target::BucketTarget {
|
||||
.map(|arn| crate::app::storage_compat::target::BucketTarget {
|
||||
arn: (*arn).to_string(),
|
||||
target_type: BucketTargetType::ReplicationService,
|
||||
..Default::default()
|
||||
|
||||
@@ -12,12 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::app::storage_compat::ecstore::{
|
||||
bucket::metadata_sys,
|
||||
disk::endpoint::Endpoint,
|
||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
store::ECStore,
|
||||
};
|
||||
use crate::app::storage_compat::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, metadata_sys};
|
||||
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager};
|
||||
use rustfs_storage_api::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _};
|
||||
@@ -82,7 +77,7 @@ async fn setup_capacity_dirty_scope_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
};
|
||||
|
||||
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
||||
crate::app::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
|
||||
crate::app::storage_compat::init_local_disks(endpoint_pools.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
|
||||
@@ -17,11 +17,11 @@ use super::handles::{
|
||||
default_bucket_metadata_interface, default_endpoints_interface, default_kms_runtime_interface,
|
||||
default_server_config_interface, default_tier_config_interface,
|
||||
};
|
||||
use crate::app::storage_compat::ecstore::bucket::metadata_sys::BucketMetadataSys;
|
||||
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools;
|
||||
use crate::app::storage_compat::ecstore::new_object_layer_fn;
|
||||
use crate::app::storage_compat::ecstore::store::ECStore;
|
||||
use crate::app::storage_compat::ecstore::tier::tier::TierConfigMgr;
|
||||
use crate::app::storage_compat::ECStore;
|
||||
use crate::app::storage_compat::EndpointServerPools;
|
||||
use crate::app::storage_compat::TierConfigMgr;
|
||||
use crate::app::storage_compat::metadata_sys::BucketMetadataSys;
|
||||
use crate::app::storage_compat::new_object_layer_fn;
|
||||
#[cfg(test)]
|
||||
use crate::config::RustFSBufferConfig;
|
||||
use rustfs_config::server_config::Config;
|
||||
@@ -126,10 +126,10 @@ mod tests {
|
||||
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
|
||||
ServerConfigInterface, TierConfigInterface,
|
||||
};
|
||||
use crate::app::storage_compat::ecstore::disk::endpoint::Endpoint;
|
||||
use crate::app::storage_compat::ecstore::endpoints::{Endpoints, PoolEndpoints};
|
||||
use crate::app::storage_compat::ecstore::new_object_layer_fn;
|
||||
use crate::app::storage_compat::ecstore::store::init_local_disks;
|
||||
use crate::app::storage_compat::Endpoint;
|
||||
use crate::app::storage_compat::init_local_disks;
|
||||
use crate::app::storage_compat::new_object_layer_fn;
|
||||
use crate::app::storage_compat::{Endpoints, PoolEndpoints};
|
||||
use crate::config::{RustFSBufferConfig, WorkloadProfile};
|
||||
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
|
||||
use std::path::PathBuf;
|
||||
|
||||
@@ -21,7 +21,7 @@ use super::interfaces::{
|
||||
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
|
||||
NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface,
|
||||
};
|
||||
use crate::app::storage_compat::ecstore::{set_object_store_resolver, store::ECStore};
|
||||
use crate::app::storage_compat::{ECStore, set_object_store_resolver};
|
||||
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
|
||||
use rustfs_kms::KmsServiceManager;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
|
||||
@@ -16,10 +16,10 @@ use super::interfaces::{
|
||||
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
|
||||
NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface,
|
||||
};
|
||||
use crate::app::storage_compat::ecstore::bucket::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys};
|
||||
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools;
|
||||
use crate::app::storage_compat::ecstore::global::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr};
|
||||
use crate::app::storage_compat::ecstore::tier::tier::TierConfigMgr;
|
||||
use crate::app::storage_compat::EndpointServerPools;
|
||||
use crate::app::storage_compat::TierConfigMgr;
|
||||
use crate::app::storage_compat::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys};
|
||||
use crate::app::storage_compat::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr};
|
||||
use crate::config::{RustFSBufferConfig, get_global_buffer_config};
|
||||
use async_trait::async_trait;
|
||||
use rustfs_config::server_config::Config;
|
||||
|
||||
@@ -12,9 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::app::storage_compat::ecstore::bucket::metadata_sys::BucketMetadataSys;
|
||||
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools;
|
||||
use crate::app::storage_compat::ecstore::tier::tier::TierConfigMgr;
|
||||
use crate::app::storage_compat::EndpointServerPools;
|
||||
use crate::app::storage_compat::TierConfigMgr;
|
||||
use crate::app::storage_compat::metadata_sys::BucketMetadataSys;
|
||||
use crate::config::RustFSBufferConfig;
|
||||
use async_trait::async_trait;
|
||||
use rustfs_config::server_config::Config;
|
||||
|
||||
@@ -14,19 +14,13 @@
|
||||
|
||||
use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase};
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::app::storage_compat::ecstore::{
|
||||
bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG},
|
||||
bucket::metadata_sys,
|
||||
client::object_api_utils::to_s3s_etag,
|
||||
client::transition_api::{ReadCloser, ReaderImpl},
|
||||
disk::endpoint::Endpoint,
|
||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
||||
global::GLOBAL_TierConfigMgr,
|
||||
store::ECStore,
|
||||
tier::{
|
||||
tier_config::{TierConfig, TierType},
|
||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||
},
|
||||
use crate::app::storage_compat::{
|
||||
ECStore, Endpoint, EndpointServerPools, Endpoints, GLOBAL_TierConfigMgr, PoolEndpoints, TierConfig, TierType, WarmBackend,
|
||||
WarmBackendGetOpts,
|
||||
metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG},
|
||||
metadata_sys,
|
||||
object_api_utils::to_s3s_etag,
|
||||
transition_api::{ReadCloser, ReaderImpl},
|
||||
};
|
||||
use crate::storage::ecfs::FS;
|
||||
use crate::storage::{
|
||||
@@ -113,7 +107,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
|
||||
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
||||
|
||||
crate::app::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
|
||||
crate::app::storage_compat::init_local_disks(endpoint_pools.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -132,7 +126,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||
|
||||
crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
|
||||
crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
|
||||
|
||||
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
|
||||
|
||||
@@ -932,7 +926,7 @@ async fn delete_transitioned_object_removes_remote_tier_copy_via_usecase() {
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||
|
||||
crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||
crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||
ecstore.clone(),
|
||||
bucket.as_str(),
|
||||
)
|
||||
@@ -992,7 +986,7 @@ async fn lifecycle_transition_marks_dirty_disks_for_capacity_manager() {
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||
|
||||
crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||
crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||
ecstore.clone(),
|
||||
bucket.as_str(),
|
||||
)
|
||||
|
||||
@@ -16,22 +16,22 @@
|
||||
|
||||
use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
|
||||
use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write};
|
||||
use crate::app::storage_compat::ecstore::bucket::quota::checker::QuotaChecker;
|
||||
use crate::app::storage_compat::ecstore::bucket::{
|
||||
use crate::app::storage_compat::ECStore;
|
||||
use crate::app::storage_compat::is_disk_compressible;
|
||||
use crate::app::storage_compat::is_valid_storage_class;
|
||||
use crate::app::storage_compat::object_api_utils::to_s3s_etag;
|
||||
use crate::app::storage_compat::quota::checker::QuotaChecker;
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_compat::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
|
||||
use crate::app::storage_compat::{HashReader, WritePlan};
|
||||
use crate::app::storage_compat::{StorageError, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::app::storage_compat::{
|
||||
lifecycle::{bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::enqueue_transition_immediate},
|
||||
metadata_sys,
|
||||
quota::QuotaOperation,
|
||||
replication::{get_must_replicate_options, must_replicate, schedule_replication},
|
||||
versioning_sys::BucketVersioningSys,
|
||||
};
|
||||
use crate::app::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
|
||||
use crate::app::storage_compat::ecstore::compress::is_disk_compressible;
|
||||
use crate::app::storage_compat::ecstore::error::{StorageError, is_err_object_not_found, is_err_version_not_found};
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_compat::ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
|
||||
use crate::app::storage_compat::ecstore::rio::{HashReader, WritePlan};
|
||||
use crate::app::storage_compat::ecstore::set_disk::is_valid_storage_class;
|
||||
use crate::app::storage_compat::ecstore::store::ECStore;
|
||||
use crate::capacity::record_capacity_write;
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::access::has_bypass_governance_header;
|
||||
@@ -479,7 +479,7 @@ impl DefaultMultipartUsecase {
|
||||
));
|
||||
}
|
||||
// Update quota tracking after successful multipart upload
|
||||
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_write_memory(
|
||||
crate::app::storage_compat::record_bucket_object_write_memory(
|
||||
&bucket,
|
||||
previous_current_size,
|
||||
obj_info.size.max(0) as u64,
|
||||
@@ -672,7 +672,7 @@ impl DefaultMultipartUsecase {
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
crate::app::storage_compat::ecstore::rio::compression_metadata_value(CompressionAlgorithm::default()),
|
||||
crate::app::storage_compat::compression_metadata_value(CompressionAlgorithm::default()),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -891,18 +891,13 @@ impl DefaultMultipartUsecase {
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
|
||||
let ssec_write = match ssec_material.key_kind {
|
||||
crate::storage::sse::EncryptionKeyKind::Object => {
|
||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
|
||||
ssec_material.key_bytes,
|
||||
part_id as u32,
|
||||
)
|
||||
}
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => {
|
||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
|
||||
ssec_material.key_bytes,
|
||||
ssec_material.base_nonce,
|
||||
part_id,
|
||||
)
|
||||
crate::app::storage_compat::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
|
||||
}
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
|
||||
ssec_material.key_bytes,
|
||||
ssec_material.base_nonce,
|
||||
part_id,
|
||||
),
|
||||
};
|
||||
write_plan = write_plan.with_encryption(ssec_write);
|
||||
(Some(ssec_material.server_side_encryption), ssec_material.kms_key_id)
|
||||
@@ -918,18 +913,13 @@ impl DefaultMultipartUsecase {
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
|
||||
let managed_write = match managed_material.key_kind {
|
||||
crate::storage::sse::EncryptionKeyKind::Object => {
|
||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
|
||||
managed_material.key_bytes,
|
||||
part_id as u32,
|
||||
)
|
||||
}
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => {
|
||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
|
||||
managed_material.key_bytes,
|
||||
managed_material.base_nonce,
|
||||
part_id,
|
||||
)
|
||||
crate::app::storage_compat::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
|
||||
}
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
|
||||
managed_material.key_bytes,
|
||||
managed_material.base_nonce,
|
||||
part_id,
|
||||
),
|
||||
};
|
||||
write_plan = write_plan.with_encryption(managed_write);
|
||||
(Some(server_side_encryption), ssekms_key_id)
|
||||
@@ -1244,18 +1234,13 @@ impl DefaultMultipartUsecase {
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
|
||||
let ssec_write = match ssec_material.key_kind {
|
||||
crate::storage::sse::EncryptionKeyKind::Object => {
|
||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
|
||||
ssec_material.key_bytes,
|
||||
part_id as u32,
|
||||
)
|
||||
}
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => {
|
||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
|
||||
ssec_material.key_bytes,
|
||||
ssec_material.base_nonce,
|
||||
part_id,
|
||||
)
|
||||
crate::app::storage_compat::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
|
||||
}
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
|
||||
ssec_material.key_bytes,
|
||||
ssec_material.base_nonce,
|
||||
part_id,
|
||||
),
|
||||
};
|
||||
write_plan = write_plan.with_encryption(ssec_write);
|
||||
(
|
||||
@@ -1275,18 +1260,13 @@ impl DefaultMultipartUsecase {
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
|
||||
let managed_write = match managed_material.key_kind {
|
||||
crate::storage::sse::EncryptionKeyKind::Object => {
|
||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
|
||||
managed_material.key_bytes,
|
||||
part_id as u32,
|
||||
)
|
||||
}
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => {
|
||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
|
||||
managed_material.key_bytes,
|
||||
managed_material.base_nonce,
|
||||
part_id,
|
||||
)
|
||||
crate::app::storage_compat::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
|
||||
}
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
|
||||
managed_material.key_bytes,
|
||||
managed_material.base_nonce,
|
||||
part_id,
|
||||
),
|
||||
};
|
||||
write_plan = write_plan.with_encryption(managed_write);
|
||||
(Some(server_side_encryption), ssekms_key_id, mp_info.user_defined.clone())
|
||||
|
||||
@@ -48,8 +48,18 @@ use metrics::{counter, histogram};
|
||||
use pin_project_lite::pin_project;
|
||||
use rustfs_object_capacity::capacity_manager::get_capacity_manager;
|
||||
// Performance metrics recording (with zero-copy-metrics integration)
|
||||
use crate::app::storage_compat::ecstore::bucket::quota::checker::QuotaChecker;
|
||||
use crate::app::storage_compat::ecstore::bucket::{
|
||||
use crate::app::storage_compat::ECStore;
|
||||
use crate::app::storage_compat::object_api_utils::to_s3s_etag;
|
||||
use crate::app::storage_compat::quota::checker::QuotaChecker;
|
||||
use crate::app::storage_compat::storageclass;
|
||||
use crate::app::storage_compat::{DiskError, is_all_buckets_not_found};
|
||||
use crate::app::storage_compat::{DynReader, HashReader, WritePlan, wrap_reader};
|
||||
use crate::app::storage_compat::{
|
||||
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||
};
|
||||
use crate::app::storage_compat::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||
use crate::app::storage_compat::{get_lock_acquire_timeout, is_valid_storage_class};
|
||||
use crate::app::storage_compat::{
|
||||
lifecycle::{
|
||||
bucket_lifecycle_audit::LcEventSrc,
|
||||
bucket_lifecycle_ops::{RestoreRequestOps, enqueue_transition_immediate, post_restore_opts},
|
||||
@@ -69,16 +79,6 @@ use crate::app::storage_compat::ecstore::bucket::{
|
||||
versioning::VersioningApi,
|
||||
versioning_sys::BucketVersioningSys,
|
||||
};
|
||||
use crate::app::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
|
||||
use crate::app::storage_compat::ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||
use crate::app::storage_compat::ecstore::config::storageclass;
|
||||
use crate::app::storage_compat::ecstore::disk::{error::DiskError, error_reduce::is_all_buckets_not_found};
|
||||
use crate::app::storage_compat::ecstore::error::{
|
||||
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||
};
|
||||
use crate::app::storage_compat::ecstore::rio::{DynReader, HashReader, WritePlan, wrap_reader};
|
||||
use crate::app::storage_compat::ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
|
||||
use crate::app::storage_compat::ecstore::store::ECStore;
|
||||
use rustfs_concurrency::GetObjectQueueSnapshot;
|
||||
use rustfs_filemeta::{
|
||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType,
|
||||
@@ -276,12 +276,12 @@ async fn enqueue_transitioned_delete_cleanup(
|
||||
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup);
|
||||
|
||||
let je = if opts.delete_prefix {
|
||||
crate::app::storage_compat::ecstore::bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(
|
||||
crate::app::storage_compat::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(
|
||||
&existing.transitioned_object,
|
||||
)
|
||||
} else {
|
||||
let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok());
|
||||
crate::app::storage_compat::ecstore::bucket::lifecycle::tier_sweeper::transitioned_delete_journal_entry(
|
||||
crate::app::storage_compat::lifecycle::tier_sweeper::transitioned_delete_journal_entry(
|
||||
version_id,
|
||||
opts.versioned,
|
||||
opts.version_suspended,
|
||||
@@ -292,10 +292,9 @@ async fn enqueue_transitioned_delete_cleanup(
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
crate::app::storage_compat::ecstore::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(store, &je)
|
||||
.await?;
|
||||
crate::app::storage_compat::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(store, &je).await?;
|
||||
|
||||
let mut expiry_state = crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
|
||||
let mut expiry_state = crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
|
||||
.write()
|
||||
.await;
|
||||
if let Err(err) = expiry_state.enqueue_tier_journal_entry(&je).await {
|
||||
@@ -1540,7 +1539,7 @@ impl DefaultObjectUsecase {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn prepare_get_object_read(
|
||||
req: &S3Request<GetObjectInput>,
|
||||
store: &crate::app::storage_compat::ecstore::store::ECStore,
|
||||
store: &crate::app::storage_compat::ECStore,
|
||||
manager: &ConcurrencyManager,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
@@ -2134,7 +2133,7 @@ impl DefaultObjectUsecase {
|
||||
insert_str(
|
||||
&mut metadata,
|
||||
SUFFIX_COMPRESSION,
|
||||
crate::app::storage_compat::ecstore::rio::compression_metadata_value(algorithm),
|
||||
crate::app::storage_compat::compression_metadata_value(algorithm),
|
||||
);
|
||||
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
|
||||
|
||||
@@ -2149,7 +2148,7 @@ impl DefaultObjectUsecase {
|
||||
insert_str(
|
||||
&mut opts.user_defined,
|
||||
SUFFIX_COMPRESSION,
|
||||
crate::app::storage_compat::ecstore::rio::compression_metadata_value(algorithm),
|
||||
crate::app::storage_compat::compression_metadata_value(algorithm),
|
||||
);
|
||||
insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string());
|
||||
|
||||
@@ -2341,7 +2340,7 @@ impl DefaultObjectUsecase {
|
||||
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
|
||||
|
||||
// Fast in-memory update for immediate quota and admin usage consistency
|
||||
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_write_memory(
|
||||
crate::app::storage_compat::record_bucket_object_write_memory(
|
||||
&bucket,
|
||||
previous_current_size,
|
||||
obj_info.size.max(0) as u64,
|
||||
@@ -3211,7 +3210,7 @@ impl DefaultObjectUsecase {
|
||||
insert_str(
|
||||
&mut compress_metadata,
|
||||
SUFFIX_COMPRESSION,
|
||||
crate::app::storage_compat::ecstore::rio::compression_metadata_value(CompressionAlgorithm::default()),
|
||||
crate::app::storage_compat::compression_metadata_value(CompressionAlgorithm::default()),
|
||||
);
|
||||
insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
|
||||
} else {
|
||||
@@ -3304,12 +3303,8 @@ impl DefaultObjectUsecase {
|
||||
|
||||
// Update quota tracking after successful copy
|
||||
if has_bucket_metadata {
|
||||
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_write_memory(
|
||||
&bucket,
|
||||
previous_current_size,
|
||||
oi.size.max(0) as u64,
|
||||
)
|
||||
.await;
|
||||
crate::app::storage_compat::record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64)
|
||||
.await;
|
||||
}
|
||||
|
||||
let raw_dest_version = oi.version_id.map(|v| v.to_string());
|
||||
@@ -3586,7 +3581,7 @@ impl DefaultObjectUsecase {
|
||||
);
|
||||
}
|
||||
let size = object_sizes[i].max(0) as u64;
|
||||
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_delete_memory(
|
||||
crate::app::storage_compat::record_bucket_object_delete_memory(
|
||||
&bucket,
|
||||
size,
|
||||
existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
|
||||
@@ -3824,7 +3819,7 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
|
||||
// Fast in-memory update for immediate quota and admin usage consistency
|
||||
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_delete_memory(
|
||||
crate::app::storage_compat::record_bucket_object_delete_memory(
|
||||
&bucket,
|
||||
obj_info.size.max(0) as u64,
|
||||
opts.version_id.is_none(),
|
||||
@@ -4757,7 +4752,7 @@ impl DefaultObjectUsecase {
|
||||
insert_str(
|
||||
&mut metadata,
|
||||
SUFFIX_COMPRESSION,
|
||||
crate::app::storage_compat::ecstore::rio::compression_metadata_value(algorithm),
|
||||
crate::app::storage_compat::compression_metadata_value(algorithm),
|
||||
);
|
||||
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
|
||||
|
||||
|
||||
@@ -12,96 +12,50 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod ecstore {
|
||||
pub(crate) use rustfs_ecstore::global::{new_object_layer_fn, set_object_store_resolver};
|
||||
|
||||
pub(crate) mod admin_server_info {
|
||||
pub(crate) use rustfs_ecstore::admin_server_info::get_server_info;
|
||||
}
|
||||
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use rustfs_ecstore::bucket::{
|
||||
bucket_target_sys, lifecycle, metadata, metadata_sys, object_lock, policy_sys, quota, replication, tagging, target,
|
||||
utils, versioning, versioning_sys,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod client {
|
||||
pub(crate) use rustfs_ecstore::client::object_api_utils;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::client::transition_api;
|
||||
}
|
||||
|
||||
pub(crate) mod compress {
|
||||
pub(crate) use rustfs_ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||
}
|
||||
|
||||
pub(crate) mod config {
|
||||
pub(crate) use rustfs_ecstore::config::storageclass;
|
||||
}
|
||||
|
||||
pub(crate) mod data_usage {
|
||||
pub(crate) use rustfs_ecstore::data_usage::{
|
||||
apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory,
|
||||
record_bucket_object_write_memory,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod disk {
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::disk::endpoint;
|
||||
pub(crate) use rustfs_ecstore::disk::{error, error_reduce};
|
||||
}
|
||||
|
||||
pub(crate) mod endpoints {
|
||||
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
|
||||
}
|
||||
|
||||
pub(crate) mod error {
|
||||
pub(crate) use rustfs_ecstore::error::{
|
||||
Error, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod global {
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
|
||||
pub(crate) use rustfs_ecstore::global::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr};
|
||||
}
|
||||
|
||||
pub(crate) mod notification_sys {
|
||||
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
|
||||
}
|
||||
|
||||
pub(crate) mod pools {
|
||||
pub(crate) use rustfs_ecstore::pools::{
|
||||
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod rio {
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader};
|
||||
pub(crate) use rustfs_ecstore::rio::{
|
||||
DynReader, HashReader, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod set_disk {
|
||||
pub(crate) use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
|
||||
}
|
||||
|
||||
pub(crate) mod store {
|
||||
pub(crate) use rustfs_ecstore::store::ECStore;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::store::init_local_disks;
|
||||
}
|
||||
|
||||
pub(crate) mod tier {
|
||||
pub(crate) use rustfs_ecstore::tier::tier;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::tier::{tier_config, warm_backend};
|
||||
}
|
||||
}
|
||||
pub(crate) use rustfs_ecstore::admin_server_info::get_server_info;
|
||||
pub(crate) use rustfs_ecstore::bucket::{
|
||||
bucket_target_sys, lifecycle, metadata, metadata_sys, object_lock, policy_sys, quota, replication, tagging, target, utils,
|
||||
versioning, versioning_sys,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::client::object_api_utils;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::client::transition_api;
|
||||
pub(crate) use rustfs_ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||
pub(crate) use rustfs_ecstore::config::storageclass;
|
||||
pub(crate) use rustfs_ecstore::data_usage::{
|
||||
apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory,
|
||||
record_bucket_object_write_memory,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
|
||||
pub(crate) use rustfs_ecstore::disk::error::DiskError;
|
||||
pub(crate) use rustfs_ecstore::disk::error_reduce::is_all_buckets_not_found;
|
||||
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
|
||||
pub(crate) use rustfs_ecstore::error::{
|
||||
Error, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
|
||||
pub(crate) use rustfs_ecstore::global::{
|
||||
get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr, new_object_layer_fn, set_object_store_resolver,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
|
||||
pub(crate) use rustfs_ecstore::pools::{
|
||||
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader};
|
||||
pub(crate) use rustfs_ecstore::rio::{
|
||||
DynReader, HashReader, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
|
||||
pub(crate) use rustfs_ecstore::store::ECStore;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::store::init_local_disks;
|
||||
pub(crate) use rustfs_ecstore::tier::tier::TierConfigMgr;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierType};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, WarmBackendGetOpts};
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage_compat::ecstore::disk::DiskAPI;
|
||||
use crate::storage_compat::DiskAPI;
|
||||
use rustfs_io_metrics::capacity_metrics::{
|
||||
record_capacity_cache_hit, record_capacity_cache_miss, record_capacity_cache_served, record_capacity_refresh_request,
|
||||
record_capacity_scan_mode,
|
||||
@@ -263,7 +263,7 @@ pub async fn init_capacity_management_for_local_disks() {
|
||||
"Capacity manager state changed"
|
||||
);
|
||||
|
||||
let disks = crate::storage_compat::ecstore::store::all_local_disk().await;
|
||||
let disks = crate::storage_compat::all_local_disk().await;
|
||||
if disks.is_empty() {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_CAPACITY,
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
#[allow(unsafe_op_in_unsafe_fn)]
|
||||
mod tests {
|
||||
use crate::config::{CommandResult, Config, Opt, TlsCommands};
|
||||
use crate::storage_compat::ecstore::disks_layout::DisksLayout;
|
||||
use crate::storage_compat::DisksLayout;
|
||||
use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_OBS_ENDPOINT, RUSTFS_REGION};
|
||||
use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY};
|
||||
use serial_test::serial;
|
||||
@@ -262,7 +262,7 @@ mod tests {
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_volumes_and_disk_layout_parsing() {
|
||||
use crate::storage_compat::ecstore::disks_layout::DisksLayout;
|
||||
use crate::storage_compat::DisksLayout;
|
||||
|
||||
// Test case 1: Single volume path
|
||||
let args = vec!["rustfs", "/data/vol1"];
|
||||
|
||||
+9
-18
@@ -51,20 +51,11 @@ use crate::init::{add_bucket_notification_configuration, init_buffer_profile_sys
|
||||
use crate::server::{ShutdownHandle, shutdown_event_notifier, start_http_server, stop_audit_system};
|
||||
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
|
||||
use crate::startup_iam::{bootstrap_or_defer_iam_init, publish_ready_for_iam_bootstrap};
|
||||
use crate::storage_compat::ecstore::store::init_lock_clients;
|
||||
use crate::storage_compat::ecstore::{
|
||||
bucket::replication::init_background_replication,
|
||||
bucket::{
|
||||
metadata_sys::init_bucket_metadata_sys,
|
||||
migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
|
||||
},
|
||||
config as ecconfig,
|
||||
endpoints::EndpointServerPools,
|
||||
global::set_global_rustfs_port,
|
||||
set_global_endpoints,
|
||||
store::ECStore,
|
||||
store::init_local_disks,
|
||||
update_erasure_type,
|
||||
use crate::storage_compat::init_lock_clients;
|
||||
use crate::storage_compat::{
|
||||
ECStore, EndpointServerPools, init as init_ecstore_config, init_background_replication, init_bucket_metadata_sys,
|
||||
init_global_config_sys, init_local_disks, set_global_endpoints, set_global_rustfs_port, try_migrate_bucket_metadata,
|
||||
try_migrate_iam_config, try_migrate_server_config, update_erasure_type,
|
||||
};
|
||||
use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
|
||||
use rustfs_credentials::init_global_action_credentials;
|
||||
@@ -333,7 +324,7 @@ impl RustFSServerBuilder {
|
||||
let region = region_str
|
||||
.parse()
|
||||
.map_err(|e| ServerError::Init(format!("invalid region '{region_str}': {e}")))?;
|
||||
crate::storage_compat::ecstore::global::set_global_region(region);
|
||||
crate::storage_compat::set_global_region(region);
|
||||
}
|
||||
|
||||
let server_port = server_addr.port();
|
||||
@@ -389,12 +380,12 @@ impl RustFSServerBuilder {
|
||||
}
|
||||
};
|
||||
|
||||
ecconfig::init();
|
||||
ecconfig::try_migrate_server_config(store.clone()).await;
|
||||
init_ecstore_config();
|
||||
try_migrate_server_config(store.clone()).await;
|
||||
|
||||
// Global config system (with retry).
|
||||
let mut retry = 0;
|
||||
while let Err(e) = ecconfig::init_global_config_sys(store.clone()).await {
|
||||
while let Err(e) = init_global_config_sys(store.clone()).await {
|
||||
retry += 1;
|
||||
if retry > 15 {
|
||||
shutdown_embedded_server();
|
||||
|
||||
+2
-2
@@ -12,8 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage_compat::ecstore::bucket::quota::QuotaError;
|
||||
use crate::storage_compat::ecstore::error::StorageError;
|
||||
use crate::storage_compat::StorageError;
|
||||
use crate::storage_compat::quota::QuotaError;
|
||||
use rustfs_storage_api::HTTPRangeError;
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
|
||||
|
||||
+2
-2
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::server::ShutdownHandle;
|
||||
use crate::storage::{process_lambda_configurations, process_queue_configurations, process_topic_configurations};
|
||||
use crate::storage_compat::ecstore::bucket::metadata_sys;
|
||||
use crate::storage_compat::metadata_sys;
|
||||
use crate::{admin, config, version};
|
||||
use rustfs_config::{
|
||||
DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK,
|
||||
@@ -157,7 +157,7 @@ fn arn_to_target_id(arn_str: &str) -> Result<rustfs_targets::arn::TargetID, Targ
|
||||
/// * `buckets` - A vector of bucket names to process
|
||||
#[instrument(skip_all)]
|
||||
pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
|
||||
let global_region = crate::storage_compat::ecstore::global::get_global_region();
|
||||
let global_region = crate::storage_compat::get_global_region();
|
||||
let region = global_region
|
||||
.as_ref()
|
||||
.filter(|r| !r.as_str().is_empty())
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store};
|
||||
use crate::app::context::resolve_server_config;
|
||||
use crate::storage_compat::ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook};
|
||||
use crate::storage_compat::{EcstoreEventArgs, register_event_dispatch_hook};
|
||||
use rustfs_notify::EventArgs as NotifyEventArgs;
|
||||
use rustfs_s3_types::EventName;
|
||||
use std::net::SocketAddr;
|
||||
|
||||
@@ -33,7 +33,7 @@ use crate::server::{
|
||||
use crate::storage;
|
||||
use crate::storage::rpc::InternodeRpcService;
|
||||
use crate::storage::tonic_service::make_server;
|
||||
use crate::storage_compat::ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
|
||||
use crate::storage_compat::{TONIC_RPC_PREFIX, verify_rpc_signature};
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, Method, Request as HttpRequest, Response};
|
||||
use hyper_util::{
|
||||
|
||||
@@ -12,9 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage_compat::ecstore::{
|
||||
config::com::{read_config, save_config},
|
||||
error::Error as StorageError,
|
||||
use crate::storage_compat::{
|
||||
EcstoreError as StorageError,
|
||||
com::{read_config, save_config},
|
||||
resolve_object_store_handle,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -14,9 +14,9 @@
|
||||
|
||||
use crate::server::{ServiceState, ServiceStateManager};
|
||||
use crate::server::{has_path_prefix, is_table_catalog_path};
|
||||
use crate::storage_compat::ecstore::global::is_dist_erasure;
|
||||
use crate::storage_compat::ecstore::global::{get_global_endpoints_opt, get_global_lock_clients};
|
||||
use crate::storage_compat::ecstore::resolve_object_store_handle;
|
||||
use crate::storage_compat::is_dist_erasure;
|
||||
use crate::storage_compat::resolve_object_store_handle;
|
||||
use crate::storage_compat::{get_global_endpoints_opt, get_global_lock_clients};
|
||||
use bytes::Bytes;
|
||||
use http::{Request as HttpRequest, Response, StatusCode};
|
||||
use http_body::Body;
|
||||
@@ -496,13 +496,10 @@ async fn collect_storage_readiness_uncached() -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
fn set_lock_quorum_status(
|
||||
online_hosts: &HashSet<String>,
|
||||
set_endpoints: &[crate::storage_compat::ecstore::disk::endpoint::Endpoint],
|
||||
) -> LockQuorumStatus {
|
||||
fn set_lock_quorum_status(online_hosts: &HashSet<String>, set_endpoints: &[crate::storage_compat::Endpoint]) -> LockQuorumStatus {
|
||||
let total_clients = set_endpoints
|
||||
.iter()
|
||||
.map(crate::storage_compat::ecstore::disk::endpoint::Endpoint::host_port)
|
||||
.map(crate::storage_compat::Endpoint::host_port)
|
||||
.filter(|host| !host.is_empty())
|
||||
.collect::<HashSet<_>>();
|
||||
let total_clients_len = total_clients.len();
|
||||
@@ -526,7 +523,7 @@ fn set_lock_quorum_status(
|
||||
}
|
||||
|
||||
fn aggregate_lock_quorum_status(
|
||||
pool_endpoints: &crate::storage_compat::ecstore::endpoints::EndpointServerPools,
|
||||
pool_endpoints: &crate::storage_compat::EndpointServerPools,
|
||||
online_hosts: &HashSet<String>,
|
||||
) -> LockQuorumStatus {
|
||||
let mut connected_clients = 0usize;
|
||||
@@ -842,8 +839,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn aggregate_lock_quorum_status_requires_each_set_to_meet_quorum() {
|
||||
use crate::storage_compat::ecstore::disk::endpoint::Endpoint;
|
||||
use crate::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::storage_compat::Endpoint;
|
||||
use crate::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
|
||||
let endpoints = vec![
|
||||
Endpoint {
|
||||
@@ -900,8 +897,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn aggregate_lock_quorum_status_fails_when_any_set_loses_quorum() {
|
||||
use crate::storage_compat::ecstore::disk::endpoint::Endpoint;
|
||||
use crate::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::storage_compat::Endpoint;
|
||||
use crate::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
|
||||
let endpoints = vec![
|
||||
Endpoint {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage_compat::ecstore::endpoints::EndpointServerPools;
|
||||
use crate::storage_compat::EndpointServerPools;
|
||||
use rustfs_config::{
|
||||
DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, ENV_RUSTFS_UNSUPPORTED_FS_POLICY, RUSTFS_UNSUPPORTED_FS_POLICY_FAIL,
|
||||
RUSTFS_UNSUPPORTED_FS_POLICY_WARN,
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use crate::app::context::{AppContext, get_global_app_context, init_global_app_context};
|
||||
use crate::server::{ServiceStateManager, publish_ready_when_runtime_ready};
|
||||
use crate::storage_compat::ecstore::store::ECStore;
|
||||
use crate::storage_compat::ECStore;
|
||||
use rustfs_common::{GlobalReadiness, SystemStage};
|
||||
use rustfs_iam::init_iam_sys;
|
||||
use rustfs_kms::KmsServiceManager;
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage_compat::ecstore::global::set_global_rustfs_port;
|
||||
use crate::storage_compat::set_global_rustfs_port;
|
||||
use crate::{
|
||||
capacity::capacity_integration::init_capacity_management,
|
||||
config::Config,
|
||||
@@ -57,7 +57,7 @@ pub async fn init_startup_listen_context(config: &Config) -> Result<StartupListe
|
||||
if let Some(region_str) = &config.region {
|
||||
region_str
|
||||
.parse::<s3s::region::Region>()
|
||||
.map(crate::storage_compat::ecstore::global::set_global_region)
|
||||
.map(crate::storage_compat::set_global_region)
|
||||
.map_err(|err| Error::other(format!("invalid region '{}': {}", region_str, err)))?;
|
||||
}
|
||||
|
||||
|
||||
@@ -12,16 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage_compat::ecstore::{
|
||||
bucket::{
|
||||
metadata_sys::init_bucket_metadata_sys,
|
||||
migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
|
||||
replication::get_global_replication_pool,
|
||||
},
|
||||
endpoints::EndpointServerPools,
|
||||
global::shutdown_background_services,
|
||||
notification_sys::new_global_notification_sys,
|
||||
store::ECStore,
|
||||
use crate::storage_compat::{
|
||||
ECStore, EndpointServerPools, get_global_replication_pool, init_bucket_metadata_sys, new_global_notification_sys,
|
||||
shutdown_background_services, try_migrate_bucket_metadata, try_migrate_iam_config,
|
||||
};
|
||||
use crate::{
|
||||
config::Config,
|
||||
@@ -597,16 +590,14 @@ where
|
||||
start_audit().await
|
||||
}
|
||||
|
||||
pub async fn init_notification_system(endpoint_pools: EndpointServerPools) -> crate::storage_compat::ecstore::error::Result<()> {
|
||||
pub async fn init_notification_system(endpoint_pools: EndpointServerPools) -> crate::storage_compat::EcstoreResult<()> {
|
||||
init_notification_system_with(|| new_global_notification_sys(endpoint_pools)).await
|
||||
}
|
||||
|
||||
async fn init_notification_system_with<InitFn, InitFuture>(
|
||||
init_notification: InitFn,
|
||||
) -> crate::storage_compat::ecstore::error::Result<()>
|
||||
async fn init_notification_system_with<InitFn, InitFuture>(init_notification: InitFn) -> crate::storage_compat::EcstoreResult<()>
|
||||
where
|
||||
InitFn: FnOnce() -> InitFuture,
|
||||
InitFuture: Future<Output = crate::storage_compat::ecstore::error::Result<()>>,
|
||||
InitFuture: Future<Output = crate::storage_compat::EcstoreResult<()>>,
|
||||
{
|
||||
init_notification().await
|
||||
}
|
||||
@@ -664,8 +655,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_system_returns_source_error() {
|
||||
let result =
|
||||
init_notification_system_with(|| async { Err(crate::storage_compat::ecstore::error::Error::FaultyDisk) }).await;
|
||||
let result = init_notification_system_with(|| async { Err(crate::storage_compat::EcstoreError::FaultyDisk) }).await;
|
||||
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
@@ -13,12 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
|
||||
use crate::storage_compat::ecstore::{
|
||||
bucket::replication::init_background_replication,
|
||||
config as ecconfig,
|
||||
endpoints::EndpointServerPools,
|
||||
set_global_endpoints,
|
||||
store::{ECStore, init_local_disks, init_lock_clients, prewarm_local_disk_id_map},
|
||||
use crate::storage_compat::{
|
||||
ECStore, EndpointServerPools, init as init_ecstore_config, init_background_replication, init_global_config_sys,
|
||||
init_local_disks, init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints, try_migrate_server_config,
|
||||
update_erasure_type,
|
||||
};
|
||||
use rustfs_common::{GlobalReadiness, SystemStage};
|
||||
@@ -147,11 +144,11 @@ pub async fn init_startup_storage_runtime(
|
||||
}
|
||||
|
||||
async fn init_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> {
|
||||
ecconfig::init();
|
||||
ecconfig::try_migrate_server_config(store.clone()).await;
|
||||
init_ecstore_config();
|
||||
try_migrate_server_config(store.clone()).await;
|
||||
|
||||
let mut retry_count = 0;
|
||||
while let Err(e) = ecconfig::init_global_config_sys(store.clone()).await {
|
||||
while let Err(e) = init_global_config_sys(store.clone()).await {
|
||||
let next_retry_count = retry_count + 1;
|
||||
error!(
|
||||
target: "rustfs::main::run",
|
||||
|
||||
@@ -18,11 +18,11 @@ use crate::error::ApiError;
|
||||
use crate::license::license_check;
|
||||
use crate::server::RemoteAddr;
|
||||
use crate::storage::request_context::RequestContext;
|
||||
use crate::storage::storage_compat::ecstore::bucket::metadata_sys;
|
||||
use crate::storage::storage_compat::ecstore::bucket::policy_sys::PolicySys;
|
||||
use crate::storage::storage_compat::ecstore::error::{StorageError, is_err_bucket_not_found};
|
||||
use crate::storage::storage_compat::ecstore::resolve_object_store_handle;
|
||||
use crate::storage::storage_compat::ecstore::store::ECStore;
|
||||
use crate::storage::storage_compat::ECStore;
|
||||
use crate::storage::storage_compat::metadata_sys;
|
||||
use crate::storage::storage_compat::policy_sys::PolicySys;
|
||||
use crate::storage::storage_compat::resolve_object_store_handle;
|
||||
use crate::storage::storage_compat::{StorageError, is_err_bucket_not_found};
|
||||
use metrics::counter;
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
|
||||
@@ -930,7 +930,7 @@ impl S3Access for FS {
|
||||
let req_info = ReqInfo {
|
||||
cred,
|
||||
is_owner,
|
||||
region: crate::storage::storage_compat::ecstore::global::get_global_region(),
|
||||
region: crate::storage::storage_compat::get_global_region(),
|
||||
request_context,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
+12
-14
@@ -21,21 +21,19 @@ use crate::storage::access::has_bypass_governance_header;
|
||||
use crate::storage::helper::OperationHelper;
|
||||
use crate::storage::options::get_opts;
|
||||
use crate::storage::s3_api::acl;
|
||||
use crate::storage::storage_compat::ecstore::{
|
||||
bucket::{
|
||||
metadata::{
|
||||
BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG,
|
||||
BUCKET_WEBSITE_CONFIG, OBJECT_LOCK_CONFIG,
|
||||
},
|
||||
metadata_sys,
|
||||
object_lock::objectlock_sys::check_retention_for_modification,
|
||||
replication::{GLOBAL_REPLICATION_STATS, ReplicationConfigurationExt},
|
||||
tagging::{decode_tags, decode_tags_to_map, encode_tags},
|
||||
utils::serialize,
|
||||
versioning::VersioningApi,
|
||||
versioning_sys::BucketVersioningSys,
|
||||
use crate::storage::storage_compat::{
|
||||
StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||
metadata::{
|
||||
BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG,
|
||||
BUCKET_WEBSITE_CONFIG, OBJECT_LOCK_CONFIG,
|
||||
},
|
||||
error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found},
|
||||
metadata_sys,
|
||||
object_lock::objectlock_sys::check_retention_for_modification,
|
||||
replication::{GLOBAL_REPLICATION_STATS, ReplicationConfigurationExt},
|
||||
tagging::{decode_tags, decode_tags_to_map, encode_tags},
|
||||
utils::serialize,
|
||||
versioning::VersioningApi,
|
||||
versioning_sys::BucketVersioningSys,
|
||||
};
|
||||
use crate::storage::{parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled};
|
||||
use crate::table_catalog;
|
||||
|
||||
@@ -16,12 +16,12 @@ use crate::config::{RustFSBufferConfig, WorkloadProfile, get_global_buffer_confi
|
||||
use crate::error::ApiError;
|
||||
use crate::server::cors;
|
||||
use crate::storage::ecfs::ListObjectUnorderedQuery;
|
||||
use crate::storage::storage_compat::ecstore::bucket::metadata_sys;
|
||||
use crate::storage::storage_compat::ecstore::bucket::metadata_sys::get_replication_config;
|
||||
use crate::storage::storage_compat::ecstore::bucket::object_lock::objectlock_sys;
|
||||
use crate::storage::storage_compat::ecstore::bucket::replication::ReplicationConfigurationExt;
|
||||
use crate::storage::storage_compat::ecstore::error::StorageError;
|
||||
use crate::storage::storage_compat::ecstore::resolve_object_store_handle;
|
||||
use crate::storage::storage_compat::StorageError;
|
||||
use crate::storage::storage_compat::metadata_sys;
|
||||
use crate::storage::storage_compat::metadata_sys::get_replication_config;
|
||||
use crate::storage::storage_compat::object_lock::objectlock_sys;
|
||||
use crate::storage::storage_compat::replication::ReplicationConfigurationExt;
|
||||
use crate::storage::storage_compat::resolve_object_store_handle;
|
||||
use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE};
|
||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||
use metrics::counter;
|
||||
@@ -739,7 +739,7 @@ pub(crate) async fn has_replication_rules(bucket: &str, objects: &[ObjectToDelet
|
||||
}
|
||||
|
||||
/// Helper function to get store and validate bucket exists
|
||||
pub(crate) async fn get_validated_store(bucket: &str) -> S3Result<Arc<crate::storage::storage_compat::ecstore::store::ECStore>> {
|
||||
pub(crate) async fn get_validated_store(bucket: &str) -> S3Result<Arc<crate::storage::storage_compat::ECStore>> {
|
||||
let Some(store) = resolve_object_store_handle() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
@@ -18,8 +18,8 @@ mod tests {
|
||||
use crate::server::cors;
|
||||
use crate::storage::ecfs::{FS, validate_object_lock_configuration_input};
|
||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||
use crate::storage::storage_compat::ecstore::bucket::{metadata::BucketMetadata, metadata_sys};
|
||||
use crate::storage::storage_compat::ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
use crate::storage::storage_compat::DEFAULT_READ_BUFFER_SIZE;
|
||||
use crate::storage::storage_compat::{metadata::BucketMetadata, metadata_sys};
|
||||
use crate::storage::{
|
||||
StorageObjectInfo as ObjectInfo, apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions,
|
||||
get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal, matches_origin_pattern, parse_etag,
|
||||
@@ -940,12 +940,12 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_validate_bucket_object_lock_enabled() {
|
||||
use crate::storage::storage_compat::ecstore::bucket::metadata::BucketMetadata;
|
||||
use crate::storage::storage_compat::ecstore::bucket::metadata_sys::set_bucket_metadata;
|
||||
use crate::storage::storage_compat::metadata::BucketMetadata;
|
||||
use crate::storage::storage_compat::metadata_sys::set_bucket_metadata;
|
||||
use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
if crate::storage::storage_compat::ecstore::bucket::metadata_sys::GLOBAL_BucketMetadataSys
|
||||
if crate::storage::storage_compat::metadata_sys::GLOBAL_BucketMetadataSys
|
||||
.get()
|
||||
.is_none()
|
||||
{
|
||||
@@ -1783,7 +1783,7 @@ mod tests {
|
||||
/// with a single-element vec value, matching the format expected by policy evaluation.
|
||||
#[test]
|
||||
fn test_object_tag_condition_key_format() {
|
||||
use crate::storage::storage_compat::ecstore::bucket::tagging::decode_tags_to_map;
|
||||
use crate::storage::storage_compat::tagging::decode_tags_to_map;
|
||||
use std::collections::HashMap;
|
||||
|
||||
let tags_str = "security=public&project=webapp&env=prod";
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::storage_compat::ecstore::store::ECStore;
|
||||
use crate::storage::storage_compat::ECStore;
|
||||
use rustfs_storage_api::ListOperations as _;
|
||||
use std::sync::Arc;
|
||||
|
||||
|
||||
@@ -29,11 +29,11 @@ pub mod timeout_wrapper;
|
||||
pub mod tonic_service;
|
||||
|
||||
pub(crate) type StorageDeletedObject = rustfs_storage_api::DeletedObject;
|
||||
pub(crate) type StorageGetObjectReader = crate::storage::storage_compat::ecstore::GetObjectReader;
|
||||
pub(crate) type StorageObjectInfo = crate::storage::storage_compat::ecstore::ObjectInfo;
|
||||
pub(crate) type StorageObjectOptions = crate::storage::storage_compat::ecstore::ObjectOptions;
|
||||
pub(crate) type StorageGetObjectReader = crate::storage::storage_compat::GetObjectReader;
|
||||
pub(crate) type StorageObjectInfo = crate::storage::storage_compat::ObjectInfo;
|
||||
pub(crate) type StorageObjectOptions = crate::storage::storage_compat::ObjectOptions;
|
||||
pub(crate) type StorageObjectToDelete = rustfs_storage_api::ObjectToDelete;
|
||||
pub(crate) type StoragePutObjReader = crate::storage::storage_compat::ecstore::PutObjReader;
|
||||
pub(crate) type StoragePutObjReader = crate::storage::storage_compat::PutObjReader;
|
||||
|
||||
#[cfg(test)]
|
||||
mod concurrent_fix_test;
|
||||
|
||||
@@ -12,9 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::storage_compat::ecstore::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::storage::storage_compat::ecstore::error::Result;
|
||||
use crate::storage::storage_compat::ecstore::error::StorageError;
|
||||
use crate::storage::storage_compat::Result;
|
||||
use crate::storage::storage_compat::StorageError;
|
||||
use crate::storage::storage_compat::versioning_sys::BucketVersioningSys;
|
||||
use http::header::{IF_MATCH, IF_NONE_MATCH};
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use rustfs_utils::http::{
|
||||
|
||||
@@ -14,10 +14,10 @@
|
||||
|
||||
use crate::server::RPC_PREFIX;
|
||||
use crate::storage::request_context::spawn_traced;
|
||||
use crate::storage::storage_compat::ecstore::disk::{DiskAPI, WalkDirOptions};
|
||||
use crate::storage::storage_compat::ecstore::rpc::verify_rpc_signature;
|
||||
use crate::storage::storage_compat::ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
use crate::storage::storage_compat::ecstore::store::find_local_disk_by_ref;
|
||||
use crate::storage::storage_compat::DEFAULT_READ_BUFFER_SIZE;
|
||||
use crate::storage::storage_compat::find_local_disk_by_ref;
|
||||
use crate::storage::storage_compat::verify_rpc_signature;
|
||||
use crate::storage::storage_compat::{DiskAPI, WalkDirOptions};
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures_util::TryStreamExt;
|
||||
use http::{HeaderMap, Method, Request, Response, StatusCode, Uri};
|
||||
|
||||
@@ -16,22 +16,12 @@ use crate::admin::service::{
|
||||
config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot},
|
||||
site_replication::reload_site_replication_runtime_state,
|
||||
};
|
||||
use crate::storage::storage_compat::ecstore::{
|
||||
admin_server_info::get_local_server_property,
|
||||
bucket::{metadata::load_bucket_metadata, metadata_sys},
|
||||
disk::{
|
||||
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
||||
UpdateMetadataOpts, error::DiskError,
|
||||
},
|
||||
get_global_lock_client,
|
||||
global::GLOBAL_TierConfigMgr,
|
||||
metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics},
|
||||
resolve_object_store_handle,
|
||||
rpc::{
|
||||
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
},
|
||||
store::{all_local_disk_path, find_local_disk_by_ref},
|
||||
use crate::storage::storage_compat::{
|
||||
CollectMetricsOpts, DeleteOptions, DiskAPI, DiskError, DiskInfoOptions, DiskStore, FileInfoVersions, GLOBAL_TierConfigMgr,
|
||||
LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, ReadMultipleReq, ReadMultipleResp,
|
||||
ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, UpdateMetadataOpts, all_local_disk_path,
|
||||
collect_local_metrics, find_local_disk_by_ref, get_global_lock_client, get_local_server_property,
|
||||
metadata::load_bucket_metadata, metadata_sys, resolve_object_store_handle,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::Stream;
|
||||
@@ -132,9 +122,7 @@ fn unimplemented_rpc(method: &str) -> Status {
|
||||
Status::unimplemented(format!("{method} is not implemented"))
|
||||
}
|
||||
|
||||
fn background_rebalance_start_error_message(
|
||||
result: crate::storage::storage_compat::ecstore::error::Result<()>,
|
||||
) -> Option<String> {
|
||||
fn background_rebalance_start_error_message(result: crate::storage::storage_compat::Result<()>) -> Option<String> {
|
||||
result.err().map(|err| format!("start_rebalance failed: {err}"))
|
||||
}
|
||||
|
||||
@@ -2378,9 +2366,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_background_rebalance_start_error_message_formats_error() {
|
||||
let message =
|
||||
background_rebalance_start_error_message(Err(crate::storage::storage_compat::ecstore::error::Error::other("boom")))
|
||||
.expect("background rebalance start failure should be formatted");
|
||||
let message = background_rebalance_start_error_message(Err(crate::storage::storage_compat::Error::other("boom")))
|
||||
.expect("background rebalance start failure should be formatted");
|
||||
|
||||
assert!(message.contains("start_rebalance failed"));
|
||||
assert!(message.contains("boom"));
|
||||
@@ -2710,7 +2697,7 @@ mod tests {
|
||||
vars.insert(PEER_RESTSIGNAL.to_string(), SERVICE_SIGNAL_RELOAD_DYNAMIC.to_string());
|
||||
vars.insert(
|
||||
PEER_RESTSUB_SYS.to_string(),
|
||||
crate::storage::storage_compat::ecstore::config::com::STORAGE_CLASS_SUB_SYS.to_string(),
|
||||
crate::storage::storage_compat::com::STORAGE_CLASS_SUB_SYS.to_string(),
|
||||
);
|
||||
|
||||
let request = Request::new(SignalServiceRequest {
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::s3_api::common::rustfs_owner;
|
||||
use crate::storage::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
|
||||
use crate::storage::storage_compat::object_api_utils::to_s3s_etag;
|
||||
use percent_encoding::percent_decode_str;
|
||||
use rustfs_storage_api::{
|
||||
BucketInfo, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||
use crate::storage::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
|
||||
use crate::storage::storage_compat::object_api_utils::to_s3s_etag;
|
||||
use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo};
|
||||
use s3s::dto::{CommonPrefix, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp};
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
@@ -191,7 +191,7 @@ mod tests {
|
||||
parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
|
||||
};
|
||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||
use crate::storage::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
|
||||
use crate::storage::storage_compat::object_api_utils::to_s3s_etag;
|
||||
use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo};
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::Timestamp;
|
||||
|
||||
@@ -69,7 +69,7 @@
|
||||
//! }
|
||||
//! ```
|
||||
|
||||
use crate::storage::storage_compat::ecstore::error::StorageError;
|
||||
use crate::storage::storage_compat::StorageError;
|
||||
use aes_gcm::{
|
||||
Aes256Gcm, Key, Nonce,
|
||||
aead::{Aead, KeyInit},
|
||||
@@ -128,8 +128,8 @@ const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE;
|
||||
const OBJECT_KEY_DERIVATION_CONTEXT: &[u8] = b"object-encryption-key generation";
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::storage_compat::ecstore::bucket::metadata_sys;
|
||||
use crate::storage::storage_compat::ecstore::error::Error;
|
||||
use crate::storage::storage_compat::Error;
|
||||
use crate::storage::storage_compat::metadata_sys;
|
||||
use rustfs_utils::http::headers::{
|
||||
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
|
||||
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT,
|
||||
@@ -745,29 +745,19 @@ pub struct ManagedSealedKey {
|
||||
}
|
||||
|
||||
impl EncryptionMaterial {
|
||||
pub fn write_encryption(
|
||||
&self,
|
||||
multipart_part_number: Option<usize>,
|
||||
) -> crate::storage::storage_compat::ecstore::rio::WriteEncryption {
|
||||
pub fn write_encryption(&self, multipart_part_number: Option<usize>) -> crate::storage::storage_compat::WriteEncryption {
|
||||
match (self.key_kind, multipart_part_number) {
|
||||
(EncryptionKeyKind::Object, Some(part_number)) => {
|
||||
crate::storage::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
|
||||
self.key_bytes,
|
||||
part_number as u32,
|
||||
)
|
||||
crate::storage::storage_compat::WriteEncryption::multipart_object_key(self.key_bytes, part_number as u32)
|
||||
}
|
||||
(EncryptionKeyKind::Object, None) => {
|
||||
crate::storage::storage_compat::ecstore::rio::WriteEncryption::singlepart_object_key(self.key_bytes)
|
||||
crate::storage::storage_compat::WriteEncryption::singlepart_object_key(self.key_bytes)
|
||||
}
|
||||
(EncryptionKeyKind::Direct, Some(part_number)) => {
|
||||
crate::storage::storage_compat::ecstore::rio::WriteEncryption::multipart(
|
||||
self.key_bytes,
|
||||
self.base_nonce,
|
||||
part_number,
|
||||
)
|
||||
crate::storage::storage_compat::WriteEncryption::multipart(self.key_bytes, self.base_nonce, part_number)
|
||||
}
|
||||
(EncryptionKeyKind::Direct, None) => {
|
||||
crate::storage::storage_compat::ecstore::rio::WriteEncryption::singlepart(self.key_bytes, self.base_nonce)
|
||||
crate::storage::storage_compat::WriteEncryption::singlepart(self.key_bytes, self.base_nonce)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,70 +12,34 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod ecstore {
|
||||
pub(crate) use rustfs_ecstore::global::{get_global_lock_client, resolve_object_store_handle};
|
||||
pub(crate) use rustfs_ecstore::admin_server_info::get_local_server_property;
|
||||
pub(crate) use rustfs_ecstore::bucket::{
|
||||
metadata, metadata_sys, object_lock, policy_sys, replication, tagging, utils, versioning, versioning_sys,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::client::object_api_utils;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::config::com;
|
||||
pub(crate) use rustfs_ecstore::disk::error::DiskError;
|
||||
pub(crate) use rustfs_ecstore::disk::{
|
||||
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
||||
UpdateMetadataOpts, WalkDirOptions,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::error::{
|
||||
Error, Result, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::global::{
|
||||
GLOBAL_TierConfigMgr, get_global_lock_client, get_global_region, resolve_object_store_handle,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
||||
pub(crate) use rustfs_ecstore::rio::WriteEncryption;
|
||||
pub(crate) use rustfs_ecstore::rpc::{
|
||||
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, verify_rpc_signature,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk_path, find_local_disk_by_ref};
|
||||
|
||||
pub(crate) mod admin_server_info {
|
||||
pub(crate) use rustfs_ecstore::admin_server_info::get_local_server_property;
|
||||
}
|
||||
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use rustfs_ecstore::bucket::{
|
||||
metadata, metadata_sys, object_lock, policy_sys, replication, tagging, utils, versioning, versioning_sys,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod client {
|
||||
pub(crate) use rustfs_ecstore::client::object_api_utils;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod config {
|
||||
pub(crate) use rustfs_ecstore::config::com;
|
||||
}
|
||||
|
||||
pub(crate) mod disk {
|
||||
pub(crate) use rustfs_ecstore::disk::{
|
||||
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
||||
UpdateMetadataOpts, WalkDirOptions, error,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod error {
|
||||
pub(crate) use rustfs_ecstore::error::{
|
||||
Error, Result, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod global {
|
||||
pub(crate) use rustfs_ecstore::global::{GLOBAL_TierConfigMgr, get_global_region};
|
||||
}
|
||||
|
||||
pub(crate) mod metrics_realtime {
|
||||
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
||||
}
|
||||
|
||||
pub(crate) mod rio {
|
||||
pub(crate) use rustfs_ecstore::rio::WriteEncryption;
|
||||
}
|
||||
|
||||
pub(crate) mod rpc {
|
||||
pub(crate) use rustfs_ecstore::rpc::{
|
||||
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, verify_rpc_signature,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod set_disk {
|
||||
pub(crate) use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||
}
|
||||
|
||||
pub(crate) mod store {
|
||||
pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk_path, find_local_disk_by_ref};
|
||||
}
|
||||
|
||||
pub(crate) type GetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
|
||||
pub(crate) type ObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||
pub(crate) type ObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
|
||||
pub(crate) type PutObjReader = rustfs_ecstore::store_api::PutObjReader;
|
||||
}
|
||||
pub(crate) type GetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
|
||||
pub(crate) type ObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||
pub(crate) type ObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
|
||||
pub(crate) type PutObjReader = rustfs_ecstore::store_api::PutObjReader;
|
||||
|
||||
@@ -12,62 +12,24 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod ecstore {
|
||||
pub(crate) use rustfs_ecstore::global::{resolve_object_store_handle, set_global_endpoints, update_erasure_type};
|
||||
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use rustfs_ecstore::bucket::{metadata, metadata_sys, migration, quota, replication};
|
||||
}
|
||||
|
||||
pub(crate) mod config {
|
||||
pub(crate) use rustfs_ecstore::config::{com, init, init_global_config_sys, try_migrate_server_config};
|
||||
}
|
||||
|
||||
pub(crate) mod disk {
|
||||
pub(crate) use rustfs_ecstore::disk::{DiskAPI, RUSTFS_META_BUCKET, endpoint};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod disks_layout {
|
||||
pub(crate) use rustfs_ecstore::disks_layout::DisksLayout;
|
||||
}
|
||||
|
||||
pub(crate) mod endpoints {
|
||||
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
|
||||
}
|
||||
|
||||
pub(crate) mod error {
|
||||
pub(crate) use rustfs_ecstore::error::{Error, Result, StorageError};
|
||||
}
|
||||
|
||||
pub(crate) mod event_notification {
|
||||
pub(crate) use rustfs_ecstore::event_notification::{EventArgs, register_event_dispatch_hook};
|
||||
}
|
||||
|
||||
pub(crate) mod global {
|
||||
pub(crate) use rustfs_ecstore::global::{
|
||||
get_global_endpoints_opt, get_global_lock_clients, get_global_region, is_dist_erasure, set_global_region,
|
||||
set_global_rustfs_port, shutdown_background_services,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod notification_sys {
|
||||
pub(crate) use rustfs_ecstore::notification_sys::new_global_notification_sys;
|
||||
}
|
||||
|
||||
pub(crate) mod rpc {
|
||||
pub(crate) use rustfs_ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
|
||||
}
|
||||
|
||||
pub(crate) mod set_disk {
|
||||
pub(crate) use rustfs_ecstore::set_disk::get_lock_acquire_timeout;
|
||||
}
|
||||
|
||||
pub(crate) mod store {
|
||||
pub(crate) use rustfs_ecstore::store::{
|
||||
ECStore, all_local_disk, init_local_disks, init_lock_clients, prewarm_local_disk_id_map,
|
||||
};
|
||||
}
|
||||
}
|
||||
pub(crate) use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
|
||||
pub(crate) use rustfs_ecstore::bucket::migration::{try_migrate_bucket_metadata, try_migrate_iam_config};
|
||||
pub(crate) use rustfs_ecstore::bucket::replication::{get_global_replication_pool, init_background_replication};
|
||||
pub(crate) use rustfs_ecstore::bucket::{metadata, metadata_sys, quota};
|
||||
pub(crate) use rustfs_ecstore::config::{com, init, init_global_config_sys, try_migrate_server_config};
|
||||
pub(crate) use rustfs_ecstore::disk::{DiskAPI, RUSTFS_META_BUCKET, endpoint::Endpoint};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::disks_layout::DisksLayout;
|
||||
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
|
||||
pub(crate) use rustfs_ecstore::error::{Error as EcstoreError, Result as EcstoreResult, StorageError};
|
||||
pub(crate) use rustfs_ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook};
|
||||
pub(crate) use rustfs_ecstore::global::{
|
||||
get_global_endpoints_opt, get_global_lock_clients, get_global_region, is_dist_erasure, resolve_object_store_handle,
|
||||
set_global_endpoints, set_global_region, set_global_rustfs_port, shutdown_background_services, update_erasure_type,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::notification_sys::new_global_notification_sys;
|
||||
pub(crate) use rustfs_ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
|
||||
pub(crate) use rustfs_ecstore::set_disk::get_lock_acquire_timeout;
|
||||
pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk, init_local_disks, init_lock_clients, prewarm_local_disk_id_map};
|
||||
|
||||
+12
-12
@@ -27,16 +27,16 @@ use std::{
|
||||
time::{Duration as StdDuration, Instant},
|
||||
};
|
||||
|
||||
use crate::storage_compat::ecstore::bucket::{
|
||||
use crate::storage_compat::RUSTFS_META_BUCKET;
|
||||
use crate::storage_compat::get_lock_acquire_timeout;
|
||||
use crate::storage_compat::{EcstoreError, StorageError};
|
||||
use crate::storage_compat::{
|
||||
metadata::{
|
||||
BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG,
|
||||
BUCKET_TABLE_RESERVED_PREFIX, table_catalog_path_hash,
|
||||
},
|
||||
metadata_sys,
|
||||
};
|
||||
use crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET;
|
||||
use crate::storage_compat::ecstore::error::{Error as EcstoreError, StorageError};
|
||||
use crate::storage_compat::ecstore::set_disk::get_lock_acquire_timeout;
|
||||
use bytes::Bytes;
|
||||
use datafusion::{
|
||||
arrow::datatypes::SchemaRef,
|
||||
@@ -7813,7 +7813,7 @@ mod tests {
|
||||
.map(|(bucket, _)| bucket.as_str())
|
||||
.collect::<BTreeSet<_>>();
|
||||
|
||||
assert_eq!(object_buckets, BTreeSet::from([crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET]));
|
||||
assert_eq!(object_buckets, BTreeSet::from([crate::storage_compat::RUSTFS_META_BUCKET]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -10777,7 +10777,7 @@ mod tests {
|
||||
.unwrap();
|
||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||
backend
|
||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &idempotency_path, 1)
|
||||
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &idempotency_path, 1)
|
||||
.await;
|
||||
|
||||
let err = store
|
||||
@@ -10827,7 +10827,7 @@ mod tests {
|
||||
.unwrap();
|
||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||
backend
|
||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
|
||||
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
|
||||
.await;
|
||||
|
||||
let request = TableCommitRequest {
|
||||
@@ -10882,7 +10882,7 @@ mod tests {
|
||||
.unwrap();
|
||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||
backend
|
||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
|
||||
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
|
||||
.await;
|
||||
|
||||
let result = store
|
||||
@@ -10936,7 +10936,7 @@ mod tests {
|
||||
.unwrap();
|
||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||
backend
|
||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
|
||||
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
|
||||
.await;
|
||||
|
||||
store
|
||||
@@ -10990,7 +10990,7 @@ mod tests {
|
||||
.unwrap();
|
||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||
backend
|
||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
|
||||
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
|
||||
.await;
|
||||
|
||||
store
|
||||
@@ -11041,7 +11041,7 @@ mod tests {
|
||||
backend.seed_object(bucket, ¤t_metadata, b"{}".to_vec()).await;
|
||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||
backend
|
||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &table_path, 2)
|
||||
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &table_path, 2)
|
||||
.await;
|
||||
|
||||
let err = store
|
||||
@@ -11098,7 +11098,7 @@ mod tests {
|
||||
.unwrap();
|
||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||
backend
|
||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &idempotency_path, 2)
|
||||
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &idempotency_path, 2)
|
||||
.await;
|
||||
|
||||
let result = store
|
||||
|
||||
@@ -65,6 +65,10 @@ STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_list_c
|
||||
STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_operation_consumer_hits.txt"
|
||||
STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE="${TMP_DIR}/store_api_object_operation_local_method_hits.txt"
|
||||
DIRECT_ECSTORE_IMPORT_HITS_FILE="${TMP_DIR}/direct_ecstore_import_hits.txt"
|
||||
TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/test_harness_nested_storage_compat_hits.txt"
|
||||
RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/rustfs_nested_storage_compat_hits.txt"
|
||||
RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/rustfs_runtime_scalar_storage_compat_hits.txt"
|
||||
RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/rustfs_runtime_secondary_storage_compat_hits.txt"
|
||||
PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE="${TMP_DIR}/production_unused_compat_allow_hits.txt"
|
||||
BROAD_STORE_API_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/broad_store_api_compat_reexport_hits.txt"
|
||||
NESTED_STORE_API_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/nested_store_api_compat_module_hits.txt"
|
||||
@@ -204,10 +208,18 @@ require_source_line \
|
||||
"crates/storage-api/src/lib.rs" \
|
||||
"pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};" \
|
||||
"storage-api public bucket contract re-export"
|
||||
require_source_line \
|
||||
"crates/storage-api/src/lib.rs" \
|
||||
"pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};" \
|
||||
"storage-api public capability contract re-export"
|
||||
require_source_line \
|
||||
"crates/storage-api/src/lib.rs" \
|
||||
"pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};" \
|
||||
"storage-api public multipart DTO re-export"
|
||||
require_source_line \
|
||||
"crates/storage-api/src/lib.rs" \
|
||||
"pub use observability::{" \
|
||||
"storage-api public observability contract re-export"
|
||||
require_source_line \
|
||||
"crates/storage-api/src/lib.rs" \
|
||||
"pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};" \
|
||||
@@ -240,6 +252,10 @@ require_source_line \
|
||||
"crates/storage-api/src/lib.rs" \
|
||||
"pub use error::{StorageErrorCode, StorageResult};" \
|
||||
"storage-api public error contract re-export"
|
||||
require_source_line \
|
||||
"crates/storage-api/src/lib.rs" \
|
||||
"pub use topology::{" \
|
||||
"storage-api public topology contract re-export"
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
@@ -403,6 +419,66 @@ if [[ -s "$DIRECT_ECSTORE_IMPORT_HITS_FILE" ]]; then
|
||||
report_failure "direct rustfs_ecstore imports outside compatibility boundaries are forbidden: $(paste -sd '; ' "$DIRECT_ECSTORE_IMPORT_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading '\bstorage_compat::ecstore\b|\bmod\s+ecstore\b' \
|
||||
crates/e2e_test/src crates/heal/tests crates/scanner/tests fuzz/fuzz_targets \
|
||||
--glob '*.rs' \
|
||||
--glob '!target/**' || true
|
||||
) >"$TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE"
|
||||
|
||||
if [[ -s "$TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE" ]]; then
|
||||
report_failure "test and fuzz storage compatibility harnesses must use direct aliases instead of nested ecstore modules: $(paste -sd '; ' "$TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading '\bstorage_compat::ecstore\b|\bmod\s+ecstore\b' \
|
||||
rustfs/src \
|
||||
--glob '*.rs' \
|
||||
--glob '!target/**' || true
|
||||
) >"$RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE" ]]; then
|
||||
report_failure "RustFS runtime storage compatibility paths must use direct aliases instead of nested ecstore modules: $(paste -sd '; ' "$RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading '\bstorage_compat::(?:store|error|global|endpoints|set_disk|rpc|metrics_realtime|notification_sys|admin_server_info|store_utils|data_usage|disks_layout|event_notification)::|\bstorage_compat::disk::(?:RUSTFS_META_BUCKET|DiskAPI|endpoint::Endpoint)\b|\bstorage_compat::\{[^}\n]*(?:store|error|global|endpoints|set_disk|rpc|metrics_realtime|notification_sys|admin_server_info|store_utils|data_usage|disks_layout|event_notification|disk)::' \
|
||||
rustfs/src \
|
||||
--glob '*.rs' \
|
||||
--glob '!target/**' || true
|
||||
) >"$RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE" ]]; then
|
||||
report_failure "RustFS runtime scalar storage compatibility paths must use direct aliases instead of secondary modules: $(paste -sd '; ' "$RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
{
|
||||
rg -n --no-heading '\bstorage_compat::(?:bucket|config|rio|client|tier|compress|disk|rebalance)::' \
|
||||
rustfs/src \
|
||||
--glob '*.rs' \
|
||||
--glob '!target/**' || true
|
||||
rg -n --no-heading '\bpub\(crate\)\s+mod\s+(?:bucket|config|rio|client|tier|compress|disk|rebalance)\b' \
|
||||
rustfs/src \
|
||||
--glob '*storage_compat.rs' \
|
||||
--glob '!target/**' || true
|
||||
find rustfs/src -type f -name '*.rs' -print0 |
|
||||
xargs -0 perl -0ne '
|
||||
while (/use\s+crate::(?:app::|admin::|storage::)?storage_compat::\{[^;]*?\b(bucket|config|rio|client|tier|compress|disk|rebalance)::/sg) {
|
||||
print "$ARGV:grouped storage_compat::$1 import\n";
|
||||
}
|
||||
'
|
||||
}
|
||||
) >"$RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE" ]]; then
|
||||
report_failure "RustFS runtime secondary storage compatibility paths must use direct aliases instead of bucket/config/rio/client/tier/compress/disk/rebalance modules: $(paste -sd '; ' "$RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --no-heading '#!\[allow\(unused_imports\)\]' \
|
||||
|
||||
Reference in New Issue
Block a user