refactor: flatten test harness storage compat aliases (#3596)

* refactor: flatten test harness storage compat aliases

* refactor: flatten rustfs storage compat aliases (#3597)

* refactor: prune runtime storage compat surface (#3598)

* refactor: flatten runtime secondary storage compat (#3599)

* docs: add scheduler placement profiling baselines (#3600)

* feat: add observability topology capability contracts (#3601)
This commit is contained in:
安正超
2026-06-19 08:30:47 +08:00
committed by GitHub
parent b1c6578df1
commit ada6f7587e
87 changed files with 1551 additions and 990 deletions
@@ -15,7 +15,7 @@
// Used by test_distributed_lock_4_nodes_grpc in lock.rs
#![allow(dead_code)]
use crate::storage_compat::ecstore::rpc::node_service_time_out_client_no_auth;
use crate::storage_compat::node_service_time_out_client_no_auth;
use async_trait::async_trait;
use rustfs_lock::{
LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
@@ -41,10 +41,7 @@ impl GrpcLockClient {
&self,
) -> Result<
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
tonic::service::interceptor::InterceptedService<
tonic::transport::Channel,
crate::storage_compat::ecstore::rpc::TonicInterceptor,
>,
tonic::service::interceptor::InterceptedService<tonic::transport::Channel, crate::storage_compat::TonicInterceptor>,
>,
> {
node_service_time_out_client_no_auth(&self.addr)
@@ -14,8 +14,8 @@
// limitations under the License.
use crate::common::workspace_root;
use crate::storage_compat::ecstore::disk::{VolumeInfo, WalkDirOptions};
use crate::storage_compat::ecstore::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
use crate::storage_compat::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
use crate::storage_compat::{VolumeInfo, WalkDirOptions};
use futures::future::join_all;
use rmp_serde::{Deserializer, Serializer};
use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
@@ -15,7 +15,7 @@
use crate::common::{
RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
};
use crate::storage_compat::ecstore::bucket::bucket_target_sys::BucketTargetSys;
use crate::storage_compat::BucketTargetSys;
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+8 -17
View File
@@ -12,22 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod ecstore {
#![allow(unused_imports)]
#![allow(dead_code, unused_imports)]
pub(crate) mod bucket {
pub(crate) mod bucket_target_sys {
pub(crate) use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
}
}
pub(crate) type BucketTargetSys = rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
pub(crate) type TonicInterceptor = rustfs_ecstore::rpc::TonicInterceptor;
pub(crate) type VolumeInfo = rustfs_ecstore::disk::VolumeInfo;
pub(crate) type WalkDirOptions = rustfs_ecstore::disk::WalkDirOptions;
pub(crate) mod disk {
pub(crate) use rustfs_ecstore::disk::{VolumeInfo, WalkDirOptions};
}
pub(crate) mod rpc {
pub(crate) use rustfs_ecstore::rpc::{
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
};
}
}
pub(crate) use rustfs_ecstore::rpc::{
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
};
+6 -24
View File
@@ -12,28 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod ecstore {
#![allow(unused_imports)]
#![allow(unused_imports)]
pub(crate) mod bucket {
pub(crate) mod metadata_sys {
pub(crate) use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
}
}
pub(crate) mod disk {
pub(crate) use rustfs_ecstore::disk::DiskStore;
pub(crate) mod endpoint {
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
}
}
pub(crate) mod endpoints {
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
}
pub(crate) mod store {
pub(crate) use rustfs_ecstore::store::{ECStore, init_local_disks};
}
}
pub(crate) use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
pub(crate) use rustfs_ecstore::disk::{DiskStore, endpoint::Endpoint};
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::store::ECStore;
pub(crate) use rustfs_ecstore::store::init_local_disks;
+3 -6
View File
@@ -16,8 +16,7 @@
mod common;
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
use crate::common::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::common::storage_compat::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks};
use std::net::SocketAddr;
use tempfile::TempDir;
use tokio_util::sync::CancellationToken;
@@ -73,12 +72,10 @@ async fn test_endpoint_index_settings() -> anyhow::Result<()> {
}
// test ECStore initialization
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone()).await?;
init_local_disks(endpoint_pools.clone()).await?;
let server_addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
let ecstore =
crate::common::storage_compat::ecstore::store::ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
.await?;
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()).await?;
println!("ECStore initialized successfully with {} pools", ecstore.pools.len());
+8 -23
View File
@@ -22,7 +22,7 @@ use rustfs_heal::heal::{
#[test]
fn test_heal_event_to_heal_request_no_panic() {
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
use crate::common::storage_compat::Endpoint;
// Test that invalid pool/set indices don't cause panic
// Create endpoint using try_from or similar method
@@ -47,7 +47,7 @@ fn test_heal_event_to_heal_request_no_panic() {
#[test]
fn test_heal_event_to_heal_request_valid_indices() {
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
use crate::common::storage_compat::Endpoint;
// Test that valid indices work correctly
let endpoint_result = Endpoint::try_from("http://localhost:9000");
@@ -192,14 +192,11 @@ fn test_heal_task_status_atomic_update() {
}
async fn get_disk_status(
&self,
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
_endpoint: &crate::common::storage_compat::Endpoint,
) -> rustfs_heal::Result<rustfs_heal::heal::storage::DiskStatus> {
Ok(rustfs_heal::heal::storage::DiskStatus::Ok)
}
async fn format_disk(
&self,
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
) -> rustfs_heal::Result<()> {
async fn format_disk(&self, _endpoint: &crate::common::storage_compat::Endpoint) -> rustfs_heal::Result<()> {
Ok(())
}
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_storage_api::BucketInfo>> {
@@ -253,10 +250,7 @@ fn test_heal_task_status_atomic_update() {
) -> rustfs_heal::Result<(Vec<String>, Option<String>, bool)> {
Ok((vec![], None, false))
}
async fn get_disk_for_resume(
&self,
_set_disk_id: &str,
) -> rustfs_heal::Result<crate::common::storage_compat::ecstore::disk::DiskStore> {
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<crate::common::storage_compat::DiskStore> {
Err(rustfs_heal::Error::other("Not implemented in mock"))
}
}
@@ -326,17 +320,11 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
Ok(Vec::new())
}
async fn get_disk_status(
&self,
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
) -> rustfs_heal::Result<DiskStatus> {
async fn get_disk_status(&self, _endpoint: &crate::common::storage_compat::Endpoint) -> rustfs_heal::Result<DiskStatus> {
Ok(DiskStatus::Ok)
}
async fn format_disk(
&self,
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
) -> rustfs_heal::Result<()> {
async fn format_disk(&self, _endpoint: &crate::common::storage_compat::Endpoint) -> rustfs_heal::Result<()> {
Ok(())
}
@@ -406,10 +394,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
Ok((Vec::new(), None, false))
}
async fn get_disk_for_resume(
&self,
_set_disk_id: &str,
) -> rustfs_heal::Result<crate::common::storage_compat::ecstore::disk::DiskStore> {
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<crate::common::storage_compat::DiskStore> {
Err(rustfs_heal::Error::other("not implemented"))
}
}
+4 -8
View File
@@ -14,10 +14,8 @@
mod common;
use crate::common::storage_compat::ecstore::{
disk::endpoint::Endpoint,
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
store::ECStore,
use crate::common::storage_compat::{
ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
};
use http::HeaderMap;
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
@@ -124,9 +122,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
// format disks (only first time)
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
.await
.unwrap();
init_local_disks(endpoint_pools.clone()).await.unwrap();
// create ECStore with dynamic port 0 (let OS assign) or fixed 9001 if free
let port = 9001; // for simplicity
@@ -144,7 +140,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
// Create heal storage layer
let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
+21 -64
View File
@@ -12,70 +12,27 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod ecstore {
#![allow(unused_imports)]
#![allow(dead_code, unused_imports)]
pub(crate) mod bucket {
pub(crate) mod lifecycle {
pub(crate) use rustfs_ecstore::bucket::lifecycle::lifecycle::TransitionOptions;
pub(crate) const BUCKET_LIFECYCLE_CONFIG: &str = rustfs_ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
pub(crate) const STORAGE_FORMAT_FILE: &str = rustfs_ecstore::disk::STORAGE_FORMAT_FILE;
pub(crate) mod bucket_lifecycle_ops {
pub(crate) use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{
enqueue_transition_for_existing_objects, init_background_expiry,
};
}
}
pub(crate) type TransitionOptions = rustfs_ecstore::bucket::lifecycle::lifecycle::TransitionOptions;
pub(crate) mod metadata {
pub(crate) use rustfs_ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
}
pub(crate) mod metadata_sys {
pub(crate) use rustfs_ecstore::bucket::metadata_sys::{get, init_bucket_metadata_sys, update};
}
pub(crate) mod versioning_sys {
pub(crate) use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
}
}
pub(crate) mod client {
pub(crate) mod transition_api {
pub(crate) use rustfs_ecstore::client::transition_api::{ReadCloser, ReaderImpl};
}
}
pub(crate) mod disk {
pub(crate) use rustfs_ecstore::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, new_disk};
pub(crate) mod endpoint {
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
}
}
pub(crate) mod endpoints {
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
}
pub(crate) mod global {
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
}
pub(crate) mod pools {
pub(crate) use rustfs_ecstore::pools::path2_bucket_object_with_base_path;
}
pub(crate) mod store {
pub(crate) use rustfs_ecstore::store::{ECStore, init_local_disks};
}
pub(crate) mod tier {
pub(crate) mod tier_config {
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierMinIO, TierType};
}
pub(crate) mod warm_backend {
pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options};
}
}
}
pub(crate) use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{
enqueue_transition_for_existing_objects, init_background_expiry,
};
pub(crate) use rustfs_ecstore::bucket::metadata_sys::{
get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata,
};
pub(crate) use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
pub(crate) use rustfs_ecstore::client::transition_api::{ReadCloser, ReaderImpl};
pub(crate) use rustfs_ecstore::disk::{DiskAPI, DiskOption, endpoint::Endpoint, new_disk};
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
pub(crate) use rustfs_ecstore::pools::path2_bucket_object_with_base_path;
pub(crate) use rustfs_ecstore::store::ECStore;
pub(crate) use rustfs_ecstore::store::init_local_disks;
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierMinIO, TierType};
pub(crate) use rustfs_ecstore::tier::warm_backend::WarmBackendGetOpts;
pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, build_transition_put_options};
@@ -14,24 +14,12 @@
mod common;
use crate::common::storage_compat::ecstore::{
bucket::metadata::BUCKET_LIFECYCLE_CONFIG,
bucket::{
lifecycle::{TransitionOptions, bucket_lifecycle_ops::enqueue_transition_for_existing_objects},
metadata_sys,
versioning_sys::BucketVersioningSys,
},
client::transition_api::{ReadCloser, ReaderImpl},
disk::endpoint::Endpoint,
disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, new_disk},
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
global::GLOBAL_TierConfigMgr,
pools::path2_bucket_object_with_base_path,
store::ECStore,
tier::{
tier_config::{TierConfig, TierMinIO, TierType},
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
},
use crate::common::storage_compat::{
BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DiskAPI, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
GLOBAL_TierConfigMgr, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, TierConfig, TierMinIO, TierType,
TransitionOptions, WarmBackend, WarmBackendGetOpts, build_transition_put_options, enqueue_transition_for_existing_objects,
get_bucket_metadata, init_background_expiry, init_bucket_metadata_sys, init_local_disks, new_disk,
path2_bucket_object_with_base_path, update_bucket_metadata,
};
use futures::FutureExt;
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
@@ -125,9 +113,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
// format disks (only first time)
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
.await
.unwrap();
init_local_disks(endpoint_pools.clone()).await.unwrap();
// create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
let port = 9002; // for simplicity
@@ -145,11 +131,10 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
// Initialize background expiry workers
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
init_background_expiry(ecstore.clone()).await;
// Store in global once lock
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
@@ -197,9 +182,7 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
};
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
.await
.unwrap();
init_local_disks(endpoint_pools.clone()).await.unwrap();
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
@@ -214,11 +197,10 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
.await
.unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
if init_expiry {
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
init_background_expiry(ecstore.clone()).await;
}
(disk_paths, ecstore)
@@ -287,7 +269,7 @@ async fn set_bucket_lifecycle(bucket_name: &str) -> Result<(), Box<dyn std::erro
</Rule>
</LifecycleConfiguration>"#;
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
Ok(())
}
@@ -311,7 +293,7 @@ async fn set_bucket_lifecycle_deletemarker(bucket_name: &str) -> Result<(), Box<
</Rule>
</LifecycleConfiguration>"#;
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
Ok(())
}
@@ -334,7 +316,7 @@ async fn set_bucket_lifecycle_delmarker_expiration(bucket_name: &str, days: i64)
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
Ok(())
}
@@ -378,7 +360,7 @@ async fn set_bucket_lifecycle_transition_with_tier(
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
Ok(())
}
@@ -613,7 +595,7 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
.await
.expect("failed to stat object metadata")
.file_type();
let lifecycle = metadata_sys::get(bucket)
let lifecycle = get_bucket_metadata(bucket)
.await
.expect("failed to load bucket metadata")
.lifecycle_config
@@ -878,7 +860,7 @@ mod serial_tests {
println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
// Verify lifecycle configuration was set
match crate::common::storage_compat::ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
match get_bucket_metadata(bucket_name.as_str()).await {
Ok(bucket_meta) => {
assert!(bucket_meta.lifecycle_config.is_some());
println!("✅ Bucket metadata retrieved successfully");
@@ -1284,8 +1266,7 @@ mod serial_tests {
"stale transitioned remote object should still exist before scanner fallback runs"
);
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
init_background_expiry(ecstore.clone()).await;
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
@@ -1346,8 +1327,7 @@ mod serial_tests {
"stale transitioned remote object should still exist before scanner cleanup runs"
);
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
init_background_expiry(ecstore.clone()).await;
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
@@ -1437,7 +1417,7 @@ mod serial_tests {
</Rule>
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
@@ -1522,7 +1502,7 @@ mod serial_tests {
</Rule>
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
@@ -1714,8 +1694,7 @@ mod serial_tests {
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
init_background_expiry(ecstore.clone()).await;
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!(
@@ -1750,7 +1729,7 @@ mod serial_tests {
</Rule>
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
@@ -1815,12 +1794,11 @@ mod serial_tests {
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>"#;
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
.await
.expect("Failed to set noncurrent lifecycle configuration");
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
.await;
init_background_expiry(ecstore.clone()).await;
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
@@ -1854,7 +1832,7 @@ mod serial_tests {
</NoncurrentVersionExpiration>
</Rule>
</LifecycleConfiguration>"#;
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
.await
.expect("Failed to set noncurrent lifecycle configuration");
@@ -1941,7 +1919,7 @@ mod serial_tests {
</Rule>
</LifecycleConfiguration>"#
);
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await
.expect("Failed to set lifecycle configuration");
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
+113
View File
@@ -0,0 +1,113 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt;
use serde::{Deserialize, Serialize};
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CapabilityState {
Supported,
Unsupported,
Disabled,
#[default]
Unknown,
}
impl CapabilityState {
pub const fn is_supported(self) -> bool {
matches!(self, Self::Supported)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CapabilityStatus {
pub state: CapabilityState,
#[serde(skip_serializing_if = "Option::is_none")]
pub reason: Option<String>,
}
impl CapabilityStatus {
pub const fn new(state: CapabilityState) -> Self {
Self { state, reason: None }
}
pub const fn supported() -> Self {
Self::new(CapabilityState::Supported)
}
pub const fn unsupported() -> Self {
Self::new(CapabilityState::Unsupported)
}
pub const fn disabled() -> Self {
Self::new(CapabilityState::Disabled)
}
pub const fn unknown() -> Self {
Self::new(CapabilityState::Unknown)
}
pub fn with_reason(mut self, reason: impl Into<String>) -> Self {
self.reason = Some(reason.into());
self
}
}
impl Default for CapabilityStatus {
fn default() -> Self {
Self::unknown()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CapabilitySnapshotError {
Unavailable,
Unsupported,
InvalidSnapshot(String),
}
impl fmt::Display for CapabilitySnapshotError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Unavailable => f.write_str("capability snapshot unavailable"),
Self::Unsupported => f.write_str("capability snapshot unsupported"),
Self::InvalidSnapshot(reason) => write!(f, "invalid capability snapshot: {reason}"),
}
}
}
impl std::error::Error for CapabilitySnapshotError {}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn capability_status_serializes_unknown_and_unsupported_states() {
let unknown = CapabilityStatus::unknown();
let unsupported = CapabilityStatus::unsupported().with_reason("target does not expose profiler");
let encoded = serde_json::to_string(&(unknown, unsupported)).expect("serialize capability statuses");
let decoded: (CapabilityStatus, CapabilityStatus) =
serde_json::from_str(&encoded).expect("deserialize capability statuses");
assert_eq!(decoded.0.state, CapabilityState::Unknown);
assert_eq!(decoded.1.state, CapabilityState::Unsupported);
assert_eq!(decoded.1.reason.as_deref(), Some("target does not expose profiler"));
assert!(!decoded.1.state.is_supported());
}
}
+11
View File
@@ -16,12 +16,16 @@
pub mod admin;
pub mod bucket;
pub mod capability;
pub mod error;
pub mod multipart;
pub mod object;
pub mod observability;
pub mod topology;
pub use admin::{DiskSetSelector, StorageAdminApi};
pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
pub use error::{StorageErrorCode, StorageResult};
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
pub use object::{DeletedObject, ObjectToDelete};
@@ -31,3 +35,10 @@ pub use object::{HealOperations, MultipartOperations, NamespaceLocking, ObjectIO
pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};
pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};
pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};
pub use observability::{
MemorySamplingState, ObservabilitySnapshot, ObservabilitySnapshotProvider, PlatformSupport, UserspaceProfilingCapability,
};
pub use topology::{
DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet, TopologySnapshot,
TopologySnapshotProvider,
};
+102
View File
@@ -0,0 +1,102 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use serde::{Deserialize, Serialize};
use crate::{CapabilitySnapshotError, CapabilityStatus};
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ObservabilitySnapshot {
pub runtime_telemetry: CapabilityStatus,
pub userspace_profiling: UserspaceProfilingCapability,
pub memory_sampling: MemorySamplingState,
pub platform: PlatformSupport,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct UserspaceProfilingCapability {
pub cpu: CapabilityStatus,
pub memory: CapabilityStatus,
pub continuous_cpu: CapabilityStatus,
pub periodic_cpu: CapabilityStatus,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MemorySamplingState {
pub process: CapabilityStatus,
pub system: CapabilityStatus,
pub cgroup: CapabilityStatus,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct PlatformSupport {
pub target_triple: Option<String>,
pub os: Option<String>,
pub arch: Option<String>,
pub allocator: CapabilityStatus,
pub ebpf: CapabilityStatus,
pub numa: CapabilityStatus,
}
#[async_trait::async_trait]
pub trait ObservabilitySnapshotProvider: Send + Sync + Debug {
async fn observability_snapshot(&self) -> Result<ObservabilitySnapshot, CapabilitySnapshotError>;
}
#[cfg(test)]
mod tests {
use super::*;
use crate::CapabilityState;
#[test]
fn observability_snapshot_preserves_unknown_and_unsupported_states() {
let snapshot = ObservabilitySnapshot {
runtime_telemetry: CapabilityStatus::unknown(),
userspace_profiling: UserspaceProfilingCapability {
cpu: CapabilityStatus::unsupported().with_reason("unsupported target"),
memory: CapabilityStatus::disabled(),
continuous_cpu: CapabilityStatus::unknown(),
periodic_cpu: CapabilityStatus::supported(),
},
memory_sampling: MemorySamplingState {
process: CapabilityStatus::supported(),
system: CapabilityStatus::supported(),
cgroup: CapabilityStatus::unknown(),
},
platform: PlatformSupport {
target_triple: Some("x86_64-unknown-linux-gnu".to_owned()),
os: Some("linux".to_owned()),
arch: Some("x86_64".to_owned()),
allocator: CapabilityStatus::supported(),
ebpf: CapabilityStatus::unknown(),
numa: CapabilityStatus::unsupported(),
},
};
let encoded = serde_json::to_string(&snapshot).expect("serialize observability snapshot");
let decoded: ObservabilitySnapshot = serde_json::from_str(&encoded).expect("deserialize observability snapshot");
assert_eq!(decoded.runtime_telemetry.state, CapabilityState::Unknown);
assert_eq!(decoded.userspace_profiling.cpu.state, CapabilityState::Unsupported);
assert_eq!(decoded.userspace_profiling.memory.state, CapabilityState::Disabled);
assert_eq!(decoded.platform.numa.state, CapabilityState::Unsupported);
assert_eq!(decoded.platform.ebpf.state, CapabilityState::Unknown);
}
}
+153
View File
@@ -0,0 +1,153 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::BTreeMap;
use std::fmt::Debug;
use serde::{Deserialize, Serialize};
use crate::{CapabilitySnapshotError, CapabilityStatus};
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TopologySnapshot {
pub pools: Vec<TopologyPool>,
pub capabilities: TopologyCapabilities,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TopologyCapabilities {
pub profiling: CapabilityStatus,
pub numa: CapabilityStatus,
pub failure_domain_labels: CapabilityStatus,
pub media_labels: CapabilityStatus,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TopologyPool {
pub pool_index: usize,
pub pool_id: Option<String>,
pub labels: TopologyLabels,
pub sets: Vec<TopologySet>,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TopologySet {
pub pool_index: usize,
pub set_index: usize,
pub set_id: Option<String>,
pub labels: TopologyLabels,
pub disks: Vec<TopologyDisk>,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TopologyDisk {
pub pool_index: usize,
pub set_index: usize,
pub disk_index: usize,
pub disk_id: Option<String>,
pub labels: TopologyLabels,
pub capabilities: DiskCapabilities,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TopologyLabels {
pub zone: Option<String>,
pub rack: Option<String>,
pub node: Option<String>,
pub media: Option<String>,
pub numa_node: Option<String>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub additional: BTreeMap<String, String>,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DiskCapabilities {
pub media_type: CapabilityStatus,
pub failure_domain: CapabilityStatus,
pub numa: CapabilityStatus,
pub profiling: CapabilityStatus,
}
#[async_trait::async_trait]
pub trait TopologySnapshotProvider: Send + Sync + Debug {
async fn topology_snapshot(&self) -> Result<TopologySnapshot, CapabilitySnapshotError>;
}
#[cfg(test)]
mod tests {
use super::*;
use crate::CapabilityState;
#[test]
fn topology_snapshot_allows_missing_and_extra_labels() {
let raw = r#"{
"pools": [{
"pool_index": 0,
"pool_id": null,
"labels": {
"additional": {
"room": "a"
}
},
"sets": [{
"pool_index": 0,
"set_index": 1,
"set_id": null,
"labels": {},
"disks": [{
"pool_index": 0,
"set_index": 1,
"disk_index": 2,
"disk_id": "disk-2",
"labels": {
"media": "ssd",
"additional": {
"slot": "nvme0"
}
},
"capabilities": {
"media_type": { "state": "supported" },
"failure_domain": { "state": "unknown" },
"numa": { "state": "unsupported", "reason": "not reported" },
"profiling": { "state": "disabled" }
}
}]
}]
}],
"capabilities": {
"profiling": { "state": "supported" },
"numa": { "state": "unknown" },
"failure_domain_labels": { "state": "supported" },
"media_labels": { "state": "supported" }
}
}"#;
let snapshot: TopologySnapshot = serde_json::from_str(raw).expect("deserialize topology snapshot");
let disk = &snapshot.pools[0].sets[0].disks[0];
assert_eq!(snapshot.pools[0].labels.zone, None);
assert_eq!(snapshot.pools[0].labels.additional.get("room").map(String::as_str), Some("a"));
assert_eq!(disk.labels.media.as_deref(), Some("ssd"));
assert_eq!(disk.labels.additional.get("slot").map(String::as_str), Some("nvme0"));
assert_eq!(disk.capabilities.numa.state, CapabilityState::Unsupported);
assert_eq!(disk.capabilities.profiling.state, CapabilityState::Disabled);
}
}
+3
View File
@@ -92,14 +92,17 @@ Required `rustfs-storage-api` public re-exports:
- `pub use admin::{DiskSetSelector, StorageAdminApi};`
- `pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};`
- `pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};`
- `pub use error::{StorageErrorCode, StorageResult};`
- `pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};`
- `pub use observability::{MemorySamplingState, ObservabilitySnapshot, ObservabilitySnapshotProvider, PlatformSupport, UserspaceProfilingCapability};`
- `pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};`
- `pub use object::{ExpirationOptions, TransitionedObject};`
- `pub use object::{HealOperations, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations};`
- `pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};`
- `pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};`
- `pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};`
- `pub use topology::{DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet, TopologySnapshot, TopologySnapshotProvider};`
ECStore must keep compile-time coverage for `StorageAdminApi`, `HealOperations`,
and the separate `NamespaceLocking` operation group.
+224 -15
View File
@@ -5,17 +5,19 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-storage-api-lifecycle-contracts`
- Baseline: stacked on `rustfs/rustfs#3594` head
(`6fc05b84e2cd22a0482adfbc042184b03f8fdfa6`).
- PR type for this branch: `pure-move`
- Runtime behavior changes: no migration behavior change expected.
- Rust code changes: move lifecycle helper DTO contracts for expiration and
transitioned object metadata into rustfs-storage-api, switch ECStore internal
consumers to direct storage-api imports, and keep ECStore old-path re-exports.
- CI/script changes: add migration guards rejecting reintroduced ECStore
lifecycle helper DTO definitions and old internal consumer imports.
- Docs changes: record the lifecycle helper pure-move slice.
- Branch: `overtrue/arch-observability-topology-contracts`
- Baseline: stacked on `origin/overtrue/arch-test-harness-compat-aliases`
after `rustfs/rustfs#3600`
(`ae6a5befcf60f2f2ebce9799ba93649032234273`).
- PR type for this branch: `contract`
- Runtime behavior changes: none.
- Rust code changes: add observability and topology capability DTO/trait
contracts to `rustfs-storage-api`.
- CI/script changes: extend migration re-export guard coverage for the new
contract exports.
- Docs changes: add
[`runtime-capability-contracts.md`](runtime-capability-contracts.md) and
record the combined PR-08/API-013 plus PR-09/API-014 contract slice.
## Phase 0 Tasks
@@ -63,6 +65,60 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
[`ecstore-config-consumer-inventory.md`](ecstore-config-consumer-inventory.md)
records the current model definitions, global accessors, persistence helpers,
consumer groups, migration risks, and do-not-change contract.
- [x] `G-011` Inventory scheduler baseline.
- Acceptance:
[`scheduler-baseline.md`](scheduler-baseline.md) records current owners for
request admission, reusable scheduler/backpressure facades, workers, scanner
budget, heal admission, and the Tokio runtime builder.
- Must preserve: no Rust source changes, no scheduler/controller contract
changes, and no runtime behavior changes.
- [x] `G-012` Inventory placement and repair invariants.
- Acceptance:
[`placement-repair-invariants.md`](placement-repair-invariants.md) records
object-to-set hashing, pool/set/disk assignment boundaries, set-aware
readiness and lock quorum, scanner budget, and heal admission preservation
gates.
- Must preserve: no placement, repair, scanner, heal, readiness, lock, or
storage metadata behavior changes.
- [x] `G-013` Inventory profiling and NUMA capabilities.
- Acceptance:
[`profiling-numa-capability-inventory.md`](profiling-numa-capability-inventory.md)
records current CPU/memory profiling, cgroup memory sampling, allocator
backend, eBPF, and NUMA capability support plus no-op fallback invariants.
- Must preserve: no startup, profiling, allocator, runtime, or platform-gate
behavior changes.
## Issue #660 Capability Contract Tasks
- [x] `PR-08/API-013` Add observability snapshot contract.
- Completed slice: add `CapabilityState`, `CapabilityStatus`,
`CapabilitySnapshotError`, `ObservabilitySnapshot`,
`UserspaceProfilingCapability`, `MemorySamplingState`,
`PlatformSupport`, and `ObservabilitySnapshotProvider` to
`rustfs-storage-api`.
- Acceptance: runtime telemetry, userspace profiling, memory sampling, and
platform support states are representable without runtime, ECStore, admin,
profiling, exporter, sidecar, eBPF, or OTEL implementation dependencies.
- Must preserve: no profiling, startup, admin route, exporter, sidecar, eBPF,
OTEL, or runtime behavior changes.
- Verification: storage-api contract tests for unknown, unsupported,
disabled, and supported capability states; focused storage-api check;
migration guard; formatting; diff hygiene; and three-expert review.
- [x] `PR-09/API-014` Add topology capability contract.
- Completed slice: add `TopologySnapshot`, `TopologyCapabilities`,
`TopologyPool`, `TopologySet`, `TopologyDisk`, `TopologyLabels`,
`DiskCapabilities`, and `TopologySnapshotProvider` to
`rustfs-storage-api`.
- Acceptance: pool, set, and disk identity fields plus optional zone, rack,
node, media, NUMA, and additional labels are representable without
`rustfs-ecstore`.
- Must preserve: no ECStore endpoint/set implementation, placement,
membership, NUMA pinning, or runtime behavior changes.
- Verification: storage-api contract tests for missing and additional labels
plus supported, unsupported, unknown, and disabled capability states;
focused storage-api check; migration guard; formatting; diff hygiene; and
three-expert review.
- [x] `TEST-PRTYPE-001` Check PR type enum consistency.
- Acceptance: `./scripts/check_architecture_migration_rules.sh` parses the
allowed PR types from [`crate-boundaries.md`](crate-boundaries.md) and fails
@@ -1296,6 +1352,87 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
migration and layer guards, formatting check, diff hygiene, risk scan, full
pre-commit, and required three-expert review passed before push.
- [x] `API-051` Flatten test harness storage compatibility aliases.
- Current branch: `overtrue/arch-test-harness-compat-aliases`.
- Current slice: flatten e2e, heal, scanner, and fuzz storage compatibility
harnesses from nested `storage_compat::ecstore` modules into direct
crate-local aliases, constants, and function imports.
- Acceptance: no e2e, heal-test, scanner-test, or fuzz-target harness file
may expose or consume nested `storage_compat::ecstore` paths, and migration
rules reject reintroducing nested test/fuzz ECStore compatibility modules.
- Must preserve: e2e bucket target/RPC/disk helper imports, heal ECStore disk
and endpoint setup, scanner lifecycle/tier/disk/storage setup, fuzz bucket
validation behavior, and fuzz path-containment validation behavior.
- Risk defense: this is test-harness and fuzz-harness import cleanup only; no
production runtime behavior, ECStore ownership, storage metadata format, or
scanner/heal lifecycle logic is changed.
- Verification: focused e2e/heal/scanner test compile, harness tests,
migration and layer guards, formatting check, diff hygiene, risk scan, full
pre-commit, and required three-expert review passed before push.
- [x] `API-052` Flatten RustFS runtime storage compatibility aliases.
- Current branch: `overtrue/arch-rustfs-storage-compat-aliases`.
- Current slice: flatten RustFS root, app, admin, and storage runtime
compatibility facades from nested `storage_compat::ecstore` modules into
direct crate-local aliases, constants, and function imports.
- Acceptance: no RustFS runtime source file may expose or consume nested
`storage_compat::ecstore` paths, and migration rules reject reintroducing
nested RustFS runtime ECStore compatibility modules.
- Must preserve: startup/config/bootstrap behavior, server readiness checks,
admin replication/rebalance/tier/config handlers, app object/bucket/
multipart usecases, storage RPC/SSE/access paths, table catalog storage
access, and existing local compatibility ownership.
- Risk defense: this is RustFS runtime import cleanup only; no production
runtime behavior, ECStore ownership, storage metadata format, object I/O,
admin authorization, or readiness semantics are changed.
- Verification: focused RustFS compile, migration and layer guards,
formatting check, diff hygiene, risk scan, full pre-commit, and required
three-expert review passed before push.
- [x] `API-053` Flatten RustFS runtime scalar storage compatibility aliases.
- Current branch: `overtrue/arch-runtime-compat-surface-prune`.
- Current slice: flatten RustFS root, app, admin, and storage runtime scalar
compatibility facades such as store, error, global, endpoints, RPC,
metrics, notification, set-disk, and data-usage paths into direct
crate-local aliases and functions.
- Acceptance: RustFS runtime source no longer consumes those scalar
compatibility surfaces through secondary modules, while higher-coupling
bucket/config/rio compatibility modules remain unchanged; migration rules
reject restoring the flattened scalar paths.
- Must preserve: startup config/bootstrap behavior, server readiness checks,
admin replication/rebalance/tier/config handlers, app object/bucket/
multipart usecases, storage RPC/SSE/access paths, table catalog storage
access, and existing ECStore concrete type ownership.
- Risk defense: this is import ownership and facade-shape cleanup only; no
production runtime behavior, ECStore ownership, storage metadata format,
object I/O, admin authorization, or readiness semantics are changed.
- Verification: focused RustFS compile, migration and layer guards,
formatting check, diff hygiene, risk scan, full pre-commit, and required
three-expert review passed before push.
- [x] `API-054` Flatten RustFS runtime secondary storage compatibility aliases.
- Current branch: `overtrue/arch-runtime-secondary-compat-flatten`.
- Current slice: flatten RustFS root, app, admin, and storage runtime
secondary compatibility modules such as bucket, config, rio, client, tier,
compress, disk, and rebalance into direct crate-local aliases, modules, and
functions.
- Acceptance: RustFS runtime source no longer consumes those compatibility
surfaces through broad secondary modules, the runtime compatibility files no
longer define those wrapper modules, and migration rules reject restoring
the flattened secondary paths.
- Must preserve: startup config/bootstrap behavior, server module-switch
config reads, embedded startup storage initialization, admin bucket/meta/
tier/rebalance/config handlers, app object/bucket/multipart usecases,
storage RPC/SSE/access paths, table catalog storage access, and ECStore
concrete type ownership.
- Risk defense: this is import ownership and facade-shape cleanup only; no
production runtime behavior, ECStore ownership, storage metadata format,
object I/O, admin authorization, tier behavior, or readiness semantics are
changed.
- Verification: focused RustFS compile, migration and layer guards,
formatting check, diff hygiene, risk scan, and required three-expert review
passed before push.
## Phase 8 Background Controller Tasks
- [x] `BGC-001` Inventory background services.
@@ -1564,7 +1701,10 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Next PRs
1. `pure-move`/`consumer-migration`: continue larger cleanup slices with the
1. `contract`/`consumer-migration`: wire read-only observability and topology
snapshots to implementation owners without changing runtime, profiling,
placement, or admin route behavior.
2. `pure-move`/`consumer-migration`: continue larger cleanup slices with the
loss-prevention guards active for remaining ECStore compatibility contracts
now that broad compatibility passthroughs are fully closed.
@@ -1572,14 +1712,65 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
| Expert | Status | Notes |
|---|---|---|
| Quality/architecture | passed | S-015 removes obsolete KMS admin policy action variants after the handler fallback cleanup; API-042/API-043/API-044/API-045/API-046/API-047/API-048/API-049/API-050 narrow notify, S3 Select, OBS, IAM, Swift, heal, scanner, RustFS runtime, test, fuzz, and lifecycle helper compatibility contracts without moving ECStore storage metadata ownership. |
| Migration preservation | passed | KMS endpoint URLs, query aliases, request bodies, response contracts, and dedicated `kms:*` authorization behavior are preserved; event builder call sites, ECStore event bridge conversion, restore event data, version IDs, metadata filtering, config read/save semantics, S3 Select store/error/buffer semantics, OBS metrics state reads, IAM config/notification/error semantics, Swift bucket metadata access, heal disk/resume/task behavior, scanner lifecycle/replication/data-usage behavior, RustFS startup/admin/app/storage runtime access, e2e/test/fuzz import behavior, lifecycle expiration/transition helper DTO field contracts, unchanged no-op handling, and remove-event behavior are preserved. |
| Testing/verification | passed | Focused compiles/tests, guards, formatting, diff hygiene, risk scan, and full `make pre-commit` passed for the current slice. |
| Quality/architecture | passed | S-015 removes obsolete KMS admin policy action variants after the handler fallback cleanup; API-042/API-043/API-044/API-045/API-046/API-047/API-048/API-049/API-050/API-051/API-052/API-053/API-054 narrow notify, S3 Select, OBS, IAM, Swift, heal, scanner, RustFS runtime, test, fuzz, lifecycle helper, harness, and RustFS runtime compatibility contracts without moving ECStore storage metadata ownership; G-011/G-012/G-013 add docs-only baselines for scheduler, placement/repair, and profiling/NUMA work; Issue #660 PR-08/PR-09 add read-only observability and topology contracts in rustfs-storage-api only. |
| Migration preservation | passed | KMS endpoint URLs, query aliases, request bodies, response contracts, and dedicated `kms:*` authorization behavior are preserved; event builder call sites, ECStore event bridge conversion, restore event data, version IDs, metadata filtering, config read/save semantics, S3 Select store/error/buffer semantics, OBS metrics state reads, IAM config/notification/error semantics, Swift bucket metadata access, heal disk/resume/task behavior, scanner lifecycle/replication/data-usage behavior, RustFS startup/admin/app/storage runtime access, e2e/test/fuzz import behavior, lifecycle expiration/transition helper DTO field contracts, flattened harness and RustFS runtime scalar/secondary alias behavior, unchanged no-op handling, remove-event behavior, scheduler/readiness/placement/profiling runtime behavior, platform gates, missing/unknown capability states, and placement/topology labels are preserved. |
| Testing/verification | passed | Focused compiles/tests, fuzz target compile, guards, formatting, diff hygiene, risk scan, and full `make pre-commit` passed for prior code slices; current Issue #660 PR-08/PR-09 contract slice uses storage-api tests/checks, migration guard, formatting, diff hygiene, and three-expert review. |
## Verification Notes
Passed before push:
- Issue #660 PR-08/PR-09 current slice:
- `cargo test -p rustfs-storage-api`: passed.
- `cargo check -p rustfs-storage-api`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- `make pre-commit`: passed.
- Three-expert review: passed.
- G-011/G-012/G-013 current slice:
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `git diff --check`: passed.
- Three-expert review: passed.
- Full `make pre-commit`: not run because this slice is documentation-only.
- API-054 current slice:
- `cargo check -p rustfs --lib`: passed.
- `cargo check --tests -p rustfs`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- Rust risk scan: passed; only existing import and path rewrites were
reviewed, with no new unwrap/expect, panic/todo/unsafe, risky casts,
ad-hoc error construction, or sensitive-token handling semantics.
- API-053 current slice:
- `cargo check -p rustfs --lib`: passed.
- `cargo check --tests -p rustfs`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- Rust risk scan: passed; only existing import and path rewrites were
reviewed, with no new unwrap/expect, panic/todo/unsafe, risky casts,
ad-hoc error construction, or sensitive-token handling semantics.
- `make pre-commit`: passed.
- API-052 current slice:
- `cargo check -p rustfs --lib`: passed.
- `cargo check --tests -p rustfs`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- Rust risk scan: passed; only existing-semantic path replacement hits were
reviewed, with no new unwrap/expect, panic/todo/unsafe, risky casts,
ad-hoc error construction, or sensitive-token handling semantics.
- `make pre-commit`: passed, including 6250 nextest tests and doctests.
- API-050 current slice:
- `cargo test -p rustfs-storage-api lifecycle_helper_defaults_preserve_existing_contracts --no-fail-fast`:
passed.
@@ -1598,6 +1789,24 @@ Passed before push:
lines.
- `make pre-commit`: passed.
- API-051 current slice:
- `cargo check --tests -p e2e_test -p rustfs-heal -p rustfs-scanner`:
passed.
- `cargo check --manifest-path fuzz/Cargo.toml --all-targets`: passed.
- `cargo test -p rustfs-heal --test endpoint_index_test test_endpoint_index_settings --no-fail-fast`:
passed.
- `cargo test -p rustfs-scanner --test lifecycle_integration_test --no-run`:
passed.
- `cargo test -p e2e_test --no-run`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- Rust risk scan: passed; only existing test `unwrap` calls were touched by
import path rewrites, with no new unwrap/expect, panic/todo/unsafe, risky
casts, ad-hoc error construction, or sensitive-token handling semantics.
- `make pre-commit`: passed.
- S-015 current slice:
- `cargo test -p rustfs-policy test_legacy_kms_admin_actions_are_rejected --no-fail-fast`:
passed.
@@ -0,0 +1,97 @@
# Placement And Repair Invariants
This inventory covers `G-012` for `rustfs/backlog#666`. It records the current
object placement, readiness, lock quorum, scanner, and repair boundaries that
later scheduler or topology work must preserve.
## Object To Set Hash Rule
Objects reach a set through `Sets::get_disks_by_key`, which calls
`get_hashed_set_index` on the object key:
- `DistributionAlgoVersion::V1` uses `crc_hash(input, set_count)`.
- `DistributionAlgoVersion::V2` and `V3` use
`sip_hash(input, set_count, format_id_bytes)`.
- The format ID is part of the V2/V3 distribution seed, so changing the seed,
object key, set count, or algorithm changes placement.
Preservation rule: every object read, write, list, heal, repair, and
decommission path that resolves a set for an existing object must preserve the
same object key and format distribution algorithm.
## Pool, Set, And Disk Assignment Boundary
Pool selection is separate from set hashing:
- Existing objects are discovered across pools and resolved to the best current
pool candidate before reads or updates continue.
- New object writes select an available pool from current per-pool free-space
inputs after suspended or rebalancing pools are skipped.
- Set selection inside a pool still uses the object-to-set hash rule above.
- Disk index assignment comes from endpoint and format metadata, not from a
scheduler decision.
Boundary rule: schedulers may influence admission, worker concurrency, or
buffer sizing, but they must not rewrite pool, set, or disk indexes.
## Readiness And Lock Quorum Boundary
Runtime readiness currently checks storage and lock health independently:
- Storage readiness requires every observed set to meet write quorum based on
the set drive count and storage class data/parity shape.
- Lock readiness aggregates per-set lock-client host quorum and fails fast if
any set loses quorum.
- Object and bucket mutations acquire namespace locks through the existing
storage lock wrappers before changing object or bucket state.
Boundary rule: readiness and lock quorum must stay set-aware. A global healthy
disk count or global connected-host count is not sufficient when any individual
set is below quorum.
## Scanner Budget Preservation
Scanner cycles are bounded by `ScannerCycleBudget`:
- Runtime budget cancels the child token after the configured duration.
- Object budget cancels after the configured object count.
- Directory budget rejects additional directories and cancels with the
directories reason.
- Partial-cycle metrics and checkpoints use the budget reason.
Preservation rule: later scheduler work can change how scan cycles are admitted
only if it preserves the budget reason, checkpoint reason, and child-token
cancellation behavior.
## Heal Admission Preservation
Scanner and background repair work enter the heal manager through explicit
admission:
- Scanner object heal requests are low priority and may be accepted, merged,
rejected as full, or dropped.
- Required/high-priority heal candidates escalate on non-admission instead of
silently disappearing.
- Heal queue admission deduplicates queued and active work unless the request
explicitly forces admission.
- Full queues can drop low-priority work or displace lower-priority work for a
higher-priority request according to current manager rules.
Preservation rule: repair scheduling changes must keep admission outcomes
observable and must not convert rejected or dropped repair work into silent
success.
## Behavior Change Gates
Any later placement or repair PR must use the following gates:
- Placement gate: prove object-to-set hashing is unchanged for existing object
keys and format algorithms.
- Pool gate: prove pool selection does not choose suspended or rebalancing
pools unless the existing path already allows it.
- Quorum gate: prove storage readiness and lock readiness remain per-set.
- Scanner gate: prove scan budget reason and checkpoint mapping remain stable.
- Heal gate: prove low-priority scanner heal, forced heal, duplicate merge, and
queue-full outcomes remain distinct.
- Rollback gate: if a new scheduler sidecar is disabled, placement and repair
must fall back to the current direct ECStore/scanner/heal behavior.
@@ -0,0 +1,81 @@
# Profiling And NUMA Capability Inventory
This inventory covers `G-013` for `rustfs/backlog#667`. It records the current
profiling, memory sampling, allocator, and NUMA baseline before optional runtime
sidecars are designed.
## Platform Support Matrix
| Capability | Current support | Current owner | Baseline recommendation |
|---|---|---|---|
| CPU pprof dump | Linux and macOS builds use `pprof`; other targets return an unsupported-platform error. | `rustfs/src/profiling.rs` | Keep CPU profiling opt-in through existing env flags and cancellation token. |
| Continuous CPU profiling | Linux and macOS builds can hold a continuous `ProfilerGuard` when enabled. | `rustfs/src/profiling.rs` | Preserve single-guard ownership and avoid starting multiple continuous guards. |
| Periodic CPU profiling | Linux and macOS builds can spawn a periodic sampling loop. | `rustfs/src/profiling.rs` | Keep the loop cancellation-driven and non-fatal. |
| Jemalloc memory pprof | Only `linux` + `gnu` + `x86_64` exposes jemalloc pprof dumping. Other supported builds return an unsupported-target error. | `rustfs/src/profiling.rs` | Treat memory pprof as optional and target-gated. |
| Periodic memory pprof | Only runs where jemalloc profiling control is available and active. | `rustfs/src/profiling.rs` | Keep inactive jemalloc as a skipped dump, not a startup failure. |
| Process/system memory sampling | Uses `rustfs_io_metrics::snapshot_process_resource_and_system` plus `sysinfo` total memory. | `rustfs/src/memory_observability.rs` | Keep sampling portable and metric-gated. |
| cgroup memory sampling | Reads Linux cgroup v2 or v1 memory files when present. Missing files produce no cgroup split. | `rustfs/src/memory_observability.rs` | Keep cgroup data opportunistic and absent-safe. |
| Allocator reclaim | Uses jemalloc backend on `linux` + `gnu` + `x86_64`; otherwise mimalloc variants. | `rustfs/src/allocator_reclaim.rs` | Keep backend detection read-only and preserve effective-force behavior. |
| eBPF | No runtime eBPF sidecar is currently wired into startup. | N/A | Treat eBPF as future optional Linux-only inventory, never as a required baseline. |
| NUMA | No NUMA placement or topology controller is currently wired into startup. | N/A | Treat NUMA as future optional capability with no-op fallback. |
## Cross-Platform Baseline
The current safe baseline is:
- Profiling is opt-in through env flags and must not make startup fatal.
- Unsupported profiling targets return structured unsupported errors or skip
startup tasks.
- Memory observability records process/system metrics and adds cgroup split
only when cgroup files exist.
- Allocator reclaim observes active HTTP, delete-tail, scanner, heal, erasure,
and GET-buffer activity before reclaiming.
- Runtime thread sizing remains owned by the Tokio runtime builder and sysinfo
core detection, not NUMA topology.
## Optional Sidecar Invariants
Future sidecars for profiling, eBPF, or NUMA must preserve these invariants:
- Sidecars must be disabled by default or target-gated until explicitly enabled.
- Unsupported targets must degrade to no-op status, not panic or fail startup.
- Sidecars must use the runtime cancellation token or an equivalent explicit
shutdown handle.
- Sidecars must not mutate Tokio worker counts after runtime creation.
- Profiling output directory fallback must stay local to profiling and must not
affect object storage paths.
- NUMA fallback must preserve current runtime thread defaults, storage set
placement, and request admission behavior.
## First Implementation Candidates
`API-013`:
- Define a read-only capability contract for profiling, cgroup memory, eBPF,
allocator backend, and NUMA availability.
- Keep the contract in a low-dependency crate and report unsupported states
explicitly.
`R-016`:
- Wire storage runtime startup to consume capability snapshots read-only.
- Do not start sidecars or mutate runtime worker ownership in the same PR.
`X-012`:
- Add an optional profiling/eBPF extension point that can observe capability
status.
- Keep unsupported targets and disabled sidecars as no-op.
`X-013`:
- Add extension tests for disabled, unsupported, and enabled capability
snapshots.
- Verify no startup fatal boundary is added for optional profiling sidecars.
`R-017`:
- If a runtime service sidecar is added later, start it from the startup service
boundary with explicit shutdown ownership.
- Preserve current service order, KMS/audit/notification fatal boundaries, and
scanner/heal startup semantics.
@@ -0,0 +1,44 @@
# Runtime Capability Contracts
This document records the `rustfs/backlog#660` PR-08 and PR-09 contract slice.
It adds read-only observability and topology snapshot shapes to
`rustfs-storage-api` without coupling the contract crate to runtime, ECStore,
admin routes, profiling, or observability implementation crates.
## Observability Snapshot Contract
`ObservabilitySnapshot` records:
- Runtime telemetry capability state.
- Userspace CPU and memory profiling capability state.
- Process, system, and cgroup memory sampling state.
- Platform support for target triple, OS, architecture, allocator, eBPF, and
NUMA capability.
Unsupported, disabled, and unknown states are represented by `CapabilityState`
instead of failing snapshot construction. The contract is intentionally read
only and does not replace existing profiling routes, telemetry APIs, exporter
pipelines, or startup behavior.
## Topology Snapshot Contract
`TopologySnapshot` records:
- Pool, set, and disk identity indexes plus optional stable IDs.
- Optional zone, rack, node, media, NUMA, and additional labels.
- Topology-wide profiling, NUMA, failure-domain label, and media-label
capability states.
- Per-disk media, failure-domain, NUMA, and profiling capability states.
Missing labels are represented as absent `Option` values. Extra topology labels
belong in the `additional` label map, so future inventory labels do not require
ECStore type leakage.
## Boundary Rules
- No `rustfs-ecstore`, `rustfs-obs`, Axum, KMS, admin route, OTEL, eBPF, or
profiling implementation dependency is added to `rustfs-storage-api`.
- No placement, membership, NUMA pinning, profiling, startup, admin route, or
exporter behavior changes are part of this contract slice.
- Providers must map implementation failures into `CapabilitySnapshotError`
before crossing the contract boundary.
+78
View File
@@ -0,0 +1,78 @@
# Scheduler Baseline Inventory
This inventory covers `G-011` for `rustfs/backlog#675`. It is a docs-only
snapshot of the current scheduling, backpressure, worker, scanner, heal, and
runtime-builder ownership. It does not define new behavior.
## Current Owners
| Surface | Current owner | Current responsibility | Migration boundary |
|---|---|---|---|
| `ConcurrencyManager` | `rustfs/src/storage/concurrency/manager.rs` | Owns the RustFS S3 read-path disk-read semaphore, I/O metrics, priority queue, storage media detection, access-pattern detection, and buffer strategy. | Keep request admission and I/O metrics behavior stable until a controller can consume the same state explicitly. |
| `SchedulerManager` | `crates/concurrency/src/scheduler.rs` | Provides a reusable facade over `rustfs-io-core::IoScheduler` and derives buffer/priority decisions from `SchedulerPolicy`. | Treat it as reusable library surface; do not assume the RustFS S3 read path has already moved to this facade. |
| `BackpressureManager` | `crates/concurrency/src/backpressure.rs` | Provides a reusable duplex-pipe backpressure facade over `rustfs-io-core::BackpressureMonitor`. | Keep pipe sizing and watermark policy separate from object-read disk semaphore admission. |
| RustFS backpressure monitor | `rustfs/src/storage/backpressure.rs` | Tracks object-pipe watermark state used by RustFS storage backpressure tests and helpers. | Preserve current state labels and watermark semantics when consolidating with reusable facades. |
| `Workers` | `crates/concurrency/src/workers.rs` | Provides cooperative worker-slot admission with `take`, `give`, and `wait`; current background workflows use it for bounded set workers. | Preserve blocking/wakeup semantics and over-release clamping. |
| Scanner cycle budget | `crates/scanner/src/scanner_budget.rs` | Cancels a child token when runtime, object-count, or directory-count budget is reached. | Preserve partial-cycle reason mapping and checkpoint accounting. |
| Heal admission | `crates/heal/src/heal/manager.rs`, `crates/heal/src/heal/channel.rs`, `rustfs_common::heal_channel` | Owns priority queue admission, duplicate merge/drop/full results, active-task tracking, retry admission, and channel responses. | Preserve low-priority scanner behavior and high-priority escalation gates. |
| Tokio runtime builder | `rustfs/src/server/runtime.rs` | Builds the multi-thread runtime from env/defaults, sets thread counts, stack, queue/event intervals, I/O event cap, thread name, and optional dial9 tracing. | Keep runtime defaults and env names stable when later startup phases move ownership. |
## Current Flow
```mermaid
flowchart TD
http["HTTP/S3 request"] --> app["app object usecase"]
app --> guard["ConcurrencyManager::track_request"]
app --> permit["disk-read semaphore permit"]
permit --> strategy["I/O queue status and buffer strategy"]
strategy --> ecstore["ECStore object/read path"]
ecstore --> setdisks["hashed set disks"]
scanner["scanner cycle"] --> budget["ScannerCycleBudget"]
budget --> folder["folder/object scan"]
folder --> healreq["heal channel request"]
healreq --> admission["HealManager admission queue"]
admission --> healworker["heal workers and retries"]
startup["startup entrypoint"] --> runtime["Tokio runtime builder"]
runtime --> services["background services"]
```
## Missing State For Later Work
`R-015` storage foundation:
- Needs a stable inventory of endpoint publication, local disk prewarm, lock
client setup, and per-set readiness state before any scheduler/controller
consumes storage topology.
- Must not infer set availability only from request-path I/O metrics.
`E-011` extension/runtime consumers:
- Need explicit ownership for runtime admission snapshots before extensions can
observe scheduler or backpressure state.
- Must not receive mutable handles to `ConcurrencyManager`, heal queues, or
scanner budget tokens.
`C-011` controller work:
- Needs desired/current/status snapshots for request admission, scanner budget,
and heal queue pressure before any controller can reconcile them.
- Must keep worker mutation explicit. Read-only status should report `None` or
no-op mutation until a reviewed worker lifecycle PR exists.
## Preservation Invariants
- Request reads must keep the same disk-read semaphore admission and active GET
accounting.
- I/O queue status and congestion metrics must remain derived from the same
permit counts.
- Scanner budget cancellation must keep its reason as runtime, objects, or
directories.
- Scanner inline-heal compatibility must continue to use asynchronous heal
admission.
- Heal duplicate admission must prefer merge semantics before full-queue
rejection.
- High-priority heal admission must still be able to displace lower-priority
queued work where the current manager allows it.
- Tokio runtime env names and fallback defaults must remain unchanged.
+1 -1
View File
@@ -4,7 +4,7 @@
mod storage_compat;
use libfuzzer_sys::fuzz_target;
use crate::storage_compat::ecstore::bucket::utils::{
use crate::storage_compat::{
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
};
@@ -12,12 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod ecstore {
pub(crate) mod bucket {
pub(crate) mod utils {
pub(crate) use rustfs_ecstore::bucket::utils::{
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
};
}
}
}
pub(crate) use rustfs_ecstore::bucket::utils::{
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
};
+1 -1
View File
@@ -4,7 +4,7 @@
mod storage_compat;
use libfuzzer_sys::fuzz_target;
use crate::storage_compat::ecstore::bucket::utils::{check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix};
use crate::storage_compat::{check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix};
use rustfs_utils::path::{clean, path_join};
use std::path::{Path, PathBuf};
@@ -12,12 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod ecstore {
pub(crate) mod bucket {
pub(crate) mod utils {
pub(crate) use rustfs_ecstore::bucket::utils::{
check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix,
};
}
}
}
pub(crate) use rustfs_ecstore::bucket::utils::{
check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix,
};
+1 -1
View File
@@ -14,7 +14,7 @@
use crate::admin::auth::authenticate_request;
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::bucket::versioning_sys::BucketVersioningSys;
use crate::admin::storage_compat::versioning_sys::BucketVersioningSys;
use crate::app::context::resolve_object_store_handle;
use crate::auth::get_condition_values;
use crate::server::{ADMIN_PREFIX, RemoteAddr};
@@ -24,7 +24,7 @@ pub(crate) async fn load_server_config_from_store() -> S3Result<Config> {
return Ok(Config::new());
};
crate::admin::storage_compat::ecstore::config::com::read_config_without_migrate(store)
crate::admin::storage_compat::com::read_config_without_migrate(store)
.await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))
}
@@ -82,7 +82,7 @@ where
return Err(s3_error!(InternalError, "server storage not initialized"));
};
let mut config = crate::admin::storage_compat::ecstore::config::com::read_config_without_migrate(store.clone())
let mut config = crate::admin::storage_compat::com::read_config_without_migrate(store.clone())
.await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
@@ -90,7 +90,7 @@ where
return Ok(());
}
crate::admin::storage_compat::ecstore::config::com::save_server_config(store, &config)
crate::admin::storage_compat::com::save_server_config(store, &config)
.await
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
+10 -12
View File
@@ -12,19 +12,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::admin::storage_compat::ecstore::bucket::utils::{deserialize, serialize};
use crate::admin::storage_compat::ecstore::{
bucket::{
metadata::{
BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG,
BucketMetadata, OBJECT_LOCK_CONFIG,
},
metadata_sys,
quota::BucketQuota,
target::BucketTargets,
use crate::admin::storage_compat::utils::{deserialize, serialize};
use crate::admin::storage_compat::{
StorageError,
metadata::{
BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG,
BucketMetadata, OBJECT_LOCK_CONFIG,
},
error::StorageError,
metadata_sys,
quota::BucketQuota,
target::BucketTargets,
};
use crate::{
admin::{
+14 -14
View File
@@ -18,12 +18,12 @@ use crate::admin::service::config::{
apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem, signal_config_snapshot_reload, signal_dynamic_config_reload,
validate_server_config,
};
use crate::admin::storage_compat::ecstore::config::com::STORAGE_CLASS_SUB_SYS;
use crate::admin::storage_compat::ecstore::config::com::{
use crate::admin::storage_compat::RUSTFS_META_BUCKET;
use crate::admin::storage_compat::com::STORAGE_CLASS_SUB_SYS;
use crate::admin::storage_compat::com::{
delete_config, read_config, read_config_without_migrate, save_config, save_server_config,
};
use crate::admin::storage_compat::ecstore::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
use crate::admin::storage_compat::ecstore::disk::RUSTFS_META_BUCKET;
use crate::admin::storage_compat::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body};
use crate::app::context::resolve_object_store_handle;
use crate::auth::{check_key_valid, get_session_token};
@@ -711,7 +711,7 @@ fn success_response(config_applied: bool) -> S3Result<S3Response<(StatusCode, Bo
Ok(S3Response::with_headers((StatusCode::OK, Body::default()), headers))
}
fn object_store() -> S3Result<std::sync::Arc<crate::admin::storage_compat::ecstore::store::ECStore>> {
fn object_store() -> S3Result<std::sync::Arc<crate::admin::storage_compat::ECStore>> {
resolve_object_store_handle().ok_or_else(|| s3_error!(InternalError, "server storage not initialized"))
}
@@ -756,7 +756,7 @@ fn config_update_sub_system(directives: &[ConfigDirective]) -> S3Result<Option<&
fn validate_config_directives(directives: &[ConfigDirective]) -> S3Result<()> {
if DEFAULT_KVS.get().is_none() {
crate::admin::storage_compat::ecstore::config::init();
crate::admin::storage_compat::init();
}
let Some(defaults) = DEFAULT_KVS.get() else {
return Err(s3_error!(InternalError, "config defaults are not initialized"));
@@ -1410,7 +1410,7 @@ fn env_help_key(sub_system: &str, key: &str) -> String {
fn default_help_postfix(sub_system: &str, key: &str) -> String {
if DEFAULT_KVS.get().is_none() {
crate::admin::storage_compat::ecstore::config::init();
crate::admin::storage_compat::init();
}
DEFAULT_KVS
@@ -1897,7 +1897,7 @@ mod tests {
#[test]
fn full_config_export_can_be_reapplied() {
crate::admin::storage_compat::ecstore::config::init();
crate::admin::storage_compat::init();
let mut original = ServerConfig::new();
apply_set_directives(
&mut original,
@@ -1944,7 +1944,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
#[test]
fn build_help_response_appends_default_value_postfix() {
crate::admin::storage_compat::ecstore::config::init();
crate::admin::storage_compat::init();
let response = build_help_response(Some("identity_openid"), Some("scopes"), false).expect("help response");
assert_eq!(response.keys_help.len(), 2);
@@ -2057,7 +2057,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
#[test]
fn render_selected_config_includes_env_override_lines() {
crate::admin::storage_compat::ecstore::config::init();
crate::admin::storage_compat::init();
temp_env::with_vars(
[
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example")),
@@ -2093,7 +2093,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
#[test]
fn render_selected_config_lists_env_only_targets() {
crate::admin::storage_compat::ecstore::config::init();
crate::admin::storage_compat::init();
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || {
let config = ServerConfig::new();
let rendered = String::from_utf8(
@@ -2116,7 +2116,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
#[test]
fn render_selected_config_supports_specific_env_only_target_queries() {
crate::admin::storage_compat::ecstore::config::init();
crate::admin::storage_compat::init();
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || {
let config = ServerConfig::new();
let rendered = String::from_utf8(
@@ -2139,7 +2139,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
#[test]
fn render_selected_config_orders_default_before_named_targets() {
crate::admin::storage_compat::ecstore::config::init();
crate::admin::storage_compat::init();
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ALPHA", Some("http://alpha.example"))], || {
let mut config = ServerConfig::new();
apply_set_directives(
@@ -2320,7 +2320,7 @@ identity_openid client_id="existing-client""#,
#[test]
fn storage_class_get_target_none_matches_full_export() {
crate::admin::storage_compat::ecstore::config::init();
crate::admin::storage_compat::init();
let mut config = ServerConfig::new();
apply_set_directives(
&mut config,
+5 -5
View File
@@ -14,8 +14,8 @@
use crate::admin::auth::{authenticate_request, validate_admin_request};
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::bucket::utils::is_valid_object_prefix;
use crate::admin::storage_compat::ecstore::store_utils::is_reserved_or_invalid_bucket;
use crate::admin::storage_compat::is_reserved_or_invalid_bucket;
use crate::admin::storage_compat::utils::is_valid_object_prefix;
use crate::app::context::resolve_object_store_handle;
use crate::server::ADMIN_PREFIX;
use crate::server::RemoteAddr;
@@ -345,10 +345,10 @@ fn should_handle_root_heal_directly(hip: &HealInitParams) -> bool {
hip.bucket.is_empty() && hip.obj_prefix.is_empty() && hip.client_token.is_empty() && !hip.force_stop
}
fn map_root_heal_status(heal_err: Option<crate::admin::storage_compat::ecstore::error::Error>) -> S3Result<()> {
fn map_root_heal_status(heal_err: Option<crate::admin::storage_compat::Error>) -> S3Result<()> {
match heal_err {
None => Ok(()),
Some(crate::admin::storage_compat::ecstore::error::StorageError::NoHealRequired) => {
Some(crate::admin::storage_compat::StorageError::NoHealRequired) => {
info!(
event = EVENT_ADMIN_RESPONSE_EMITTED,
component = LOG_COMPONENT_ADMIN_API,
@@ -703,7 +703,7 @@ mod tests {
encode_heal_task_status, heal_channel_response_items, heal_channel_response_summary, json_response, map_heal_response,
map_root_heal_status, should_handle_root_heal_directly, validate_heal_request_mode, validate_heal_target,
};
use crate::admin::storage_compat::ecstore::error::StorageError;
use crate::admin::storage_compat::StorageError;
use bytes::Bytes;
use http::StatusCode;
use http::Uri;
+1 -1
View File
@@ -16,7 +16,7 @@
use crate::admin::auth::validate_admin_request;
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::config::com::{read_config, save_config};
use crate::admin::storage_compat::com::{read_config, save_config};
use crate::app::context::{resolve_kms_runtime_service_manager, resolve_object_store_handle};
use crate::auth::{check_key_valid, get_session_token};
use crate::server::{ADMIN_PREFIX, RemoteAddr};
+1 -1
View File
@@ -20,7 +20,7 @@
use crate::admin::auth::validate_admin_request;
use crate::admin::router::Operation;
use crate::admin::storage_compat::ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
use crate::admin::storage_compat::{CollectMetricsOpts, MetricType, collect_local_metrics};
use crate::auth::{check_key_valid, get_session_token};
use crate::server::RemoteAddr;
use crate::storage::request_context::spawn_traced;
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::global::get_global_region;
use crate::admin::storage_compat::get_global_region;
use crate::app::context::resolve_object_store_handle;
use crate::auth::{check_key_valid, get_session_token};
use crate::error::ApiError;
@@ -649,7 +649,7 @@ async fn preflight_zip_items(request: &CreateObjectZipDownloadRequest, items: &[
Ok(())
}
fn storage_error_to_s3(err: crate::admin::storage_compat::ecstore::error::Error) -> s3s::S3Error {
fn storage_error_to_s3(err: crate::admin::storage_compat::Error) -> s3s::S3Error {
ApiError::from(err).into()
}
+1 -1
View File
@@ -15,7 +15,7 @@
use super::sts::create_oidc_sts_credentials;
use crate::admin::auth::validate_admin_request;
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::config::com::{read_config_without_migrate, save_server_config};
use crate::admin::storage_compat::com::{read_config_without_migrate, save_server_config};
use crate::app::context::resolve_object_store_handle;
use crate::auth::{check_key_valid, get_session_token};
use crate::server::{ADMIN_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr};
+1 -1
View File
@@ -99,7 +99,7 @@ macro_rules! log_pool_response_emitted {
};
}
fn endpoints_from_context() -> Option<crate::admin::storage_compat::ecstore::endpoints::EndpointServerPools> {
fn endpoints_from_context() -> Option<crate::admin::storage_compat::EndpointServerPools> {
resolve_endpoints_handle()
}
+4 -4
View File
@@ -16,9 +16,9 @@
use crate::admin::auth::{validate_admin_request, validate_admin_request_with_bucket};
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::bucket::metadata_sys::BucketMetadataSys;
use crate::admin::storage_compat::ecstore::bucket::quota::checker::QuotaChecker;
use crate::admin::storage_compat::ecstore::bucket::quota::{BucketQuota, QuotaError, QuotaOperation};
use crate::admin::storage_compat::metadata_sys::BucketMetadataSys;
use crate::admin::storage_compat::quota::checker::QuotaChecker;
use crate::admin::storage_compat::quota::{BucketQuota, QuotaError, QuotaOperation};
use crate::app::context::{resolve_bucket_metadata_handle, resolve_object_store_handle};
use crate::auth::{check_key_valid, get_session_token};
use crate::server::ADMIN_PREFIX;
@@ -175,7 +175,7 @@ async fn current_usage_from_context(bucket: &str) -> u64 {
return 0;
};
match crate::admin::storage_compat::ecstore::data_usage::load_data_usage_from_backend(store).await {
match crate::admin::storage_compat::load_data_usage_from_backend(store).await {
Ok(data_usage_info) => data_usage_info
.buckets_usage
.get(bucket)
+5 -9
View File
@@ -12,10 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::admin::storage_compat::ecstore::{
error::StorageError,
notification_sys::get_global_notification_sys,
rebalance::{DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta},
use crate::admin::storage_compat::{
DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, StorageError, get_global_notification_sys,
};
use crate::{
admin::{
@@ -150,7 +148,7 @@ fn build_rebalance_pool_progress(
now: OffsetDateTime,
stop_time: Option<OffsetDateTime>,
percent_free_goal: f64,
ps: &crate::admin::storage_compat::ecstore::rebalance::RebalanceStats,
ps: &crate::admin::storage_compat::RebalanceStats,
) -> Option<RebalPoolProgress> {
let total_bytes_to_rebal = ps.init_capacity as f64 * percent_free_goal - ps.init_free_space as f64;
let terminal_time = ps.info.end_time.or(stop_time);
@@ -193,7 +191,7 @@ fn build_rebalance_pool_statuses(
now: OffsetDateTime,
stop_time: Option<OffsetDateTime>,
percent_free_goal: f64,
pool_stats: &[crate::admin::storage_compat::ecstore::rebalance::RebalanceStats],
pool_stats: &[crate::admin::storage_compat::RebalanceStats],
disk_stats: &[DiskStat],
) -> Vec<RebalancePoolStatus> {
pool_stats
@@ -563,9 +561,7 @@ mod rebalance_handler_tests {
RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, build_rebalance_pool_statuses, rebalance_pool_used,
rebalance_remaining_buckets, rebalance_used_pct,
};
use crate::admin::storage_compat::ecstore::rebalance::{
DiskStat, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceStats,
};
use crate::admin::storage_compat::{DiskStat, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceStats};
use time::OffsetDateTime;
#[test]
+9 -9
View File
@@ -15,15 +15,15 @@
use crate::admin::auth::validate_admin_request;
use crate::admin::handlers::site_replication::site_replication_peer_deployment_id_for_endpoint;
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::{BucketTargetError, BucketTargetSys};
use crate::admin::storage_compat::ecstore::bucket::metadata::BUCKET_TARGETS_FILE;
use crate::admin::storage_compat::ecstore::bucket::metadata_sys;
use crate::admin::storage_compat::ecstore::bucket::metadata_sys::get_replication_config;
use crate::admin::storage_compat::ecstore::bucket::replication::BucketStats;
use crate::admin::storage_compat::ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
use crate::admin::storage_compat::ecstore::bucket::target::BucketTarget;
use crate::admin::storage_compat::ecstore::error::StorageError;
use crate::admin::storage_compat::ecstore::global::global_rustfs_port;
use crate::admin::storage_compat::StorageError;
use crate::admin::storage_compat::bucket_target_sys::{BucketTargetError, BucketTargetSys};
use crate::admin::storage_compat::global_rustfs_port;
use crate::admin::storage_compat::metadata::BUCKET_TARGETS_FILE;
use crate::admin::storage_compat::metadata_sys;
use crate::admin::storage_compat::metadata_sys::get_replication_config;
use crate::admin::storage_compat::replication::BucketStats;
use crate::admin::storage_compat::replication::GLOBAL_REPLICATION_STATS;
use crate::admin::storage_compat::target::BucketTarget;
use crate::admin::utils::read_compatible_admin_body;
use crate::app::context::resolve_object_store_handle;
use crate::auth::{check_key_valid, get_session_token};
+15 -19
View File
@@ -18,24 +18,20 @@ use crate::admin::site_replication_identity::{
canonical_endpoint, deployment_id_for_endpoint, normalize_peer_map_by_identity_with, same_identity_endpoint,
site_identity_key,
};
use crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::BucketTargetSys;
use crate::admin::storage_compat::ecstore::bucket::metadata::{
use crate::admin::storage_compat::Error as StorageError;
use crate::admin::storage_compat::bucket_target_sys::BucketTargetSys;
use crate::admin::storage_compat::com::{delete_config, read_config, save_config};
use crate::admin::storage_compat::metadata::{
BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG,
BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG,
};
use crate::admin::storage_compat::ecstore::bucket::metadata_sys;
use crate::admin::storage_compat::ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
use crate::admin::storage_compat::ecstore::bucket::replication::{
ReplicationConfigurationExt, ResyncOpts, get_global_replication_pool,
};
use crate::admin::storage_compat::ecstore::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials};
use crate::admin::storage_compat::ecstore::bucket::utils::{deserialize, serialize};
use crate::admin::storage_compat::ecstore::bucket::versioning::VersioningApi;
use crate::admin::storage_compat::ecstore::config::com::{delete_config, read_config, save_config};
use crate::admin::storage_compat::ecstore::error::Error as StorageError;
use crate::admin::storage_compat::ecstore::global::{
get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port,
};
use crate::admin::storage_compat::metadata_sys;
use crate::admin::storage_compat::replication::GLOBAL_REPLICATION_STATS;
use crate::admin::storage_compat::replication::{ReplicationConfigurationExt, ResyncOpts, get_global_replication_pool};
use crate::admin::storage_compat::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials};
use crate::admin::storage_compat::utils::{deserialize, serialize};
use crate::admin::storage_compat::versioning::VersioningApi;
use crate::admin::storage_compat::{get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port};
use crate::admin::utils::{encode_compatible_admin_payload, read_compatible_admin_body};
use crate::app::context::resolve_object_store_handle;
use crate::auth::{check_key_valid, get_session_token};
@@ -656,7 +652,7 @@ async fn site_replication_peer_client() -> S3Result<reqwest::Client> {
built
}
fn runtime_tls_enabled_with(endpoints: Option<&crate::admin::storage_compat::ecstore::endpoints::EndpointServerPools>) -> bool {
fn runtime_tls_enabled_with(endpoints: Option<&crate::admin::storage_compat::EndpointServerPools>) -> bool {
if !rustfs_utils::get_env_str(ENV_RUSTFS_TLS_PATH, DEFAULT_RUSTFS_TLS_PATH).is_empty() {
return true;
}
@@ -3357,7 +3353,7 @@ fn is_stale_update(local_updated_at: OffsetDateTime, incoming_updated_at: Option
}
fn bucket_meta_local_updated_at(
bucket_meta: &crate::admin::storage_compat::ecstore::bucket::metadata::BucketMetadata,
bucket_meta: &crate::admin::storage_compat::metadata::BucketMetadata,
config_file: &str,
) -> OffsetDateTime {
match config_file {
@@ -4578,8 +4574,8 @@ impl Operation for SRRotateServiceAccountHandler {
#[cfg(test)]
mod tests {
use super::*;
use crate::admin::storage_compat::ecstore::disk::endpoint::Endpoint;
use crate::admin::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::admin::storage_compat::Endpoint;
use crate::admin::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints};
use http::{HeaderMap, HeaderValue, Uri};
use rustfs_common::{get_global_outbound_tls_generation, set_global_outbound_tls_generation};
use rustfs_policy::policy::action::S3Action;
+1 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use super::is_admin::IsAdminHandler;
use crate::admin::storage_compat::ecstore::bucket::utils::serialize;
use crate::admin::storage_compat::utils::serialize;
use crate::{
admin::{
handlers::site_replication::site_replication_iam_change_hook,
+1 -4
View File
@@ -12,10 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::admin::storage_compat::ecstore::{
bucket::{metadata::table_catalog_path_hash, metadata_sys},
store::ECStore,
};
use crate::admin::storage_compat::{ECStore, metadata::table_catalog_path_hash, metadata_sys};
use crate::admin::{
auth::{AdminResourceScope, validate_admin_request, validate_admin_request_with_bucket_object},
router::{AdminOperation, Operation, S3Router},
+6 -16
View File
@@ -13,21 +13,11 @@
// limitations under the License.
#![allow(unused_variables, unused_mut, unused_must_use)]
use crate::admin::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState;
use crate::admin::storage_compat::ecstore::{
bucket::lifecycle::tier_last_day_stats::DailyAllTierStats,
client::admin_handler_utils::AdminError,
config::storageclass,
notification_sys::get_global_notification_sys,
tier::{
tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_MISSING_CREDENTIALS},
tier_admin::TierCreds,
tier_config::{TierConfig, TierType},
tier_handlers::{
ERR_TIER_ALREADY_EXISTS, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE,
ERR_TIER_NOT_FOUND,
},
},
use crate::admin::storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState;
use crate::admin::storage_compat::{
AdminError, DailyAllTierStats, ERR_TIER_ALREADY_EXISTS, ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY,
ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE,
ERR_TIER_NOT_FOUND, TierConfig, TierCreds, TierType, get_global_notification_sys, storageclass,
};
use crate::{
admin::{
@@ -916,7 +906,7 @@ impl Operation for ClearTier {
#[cfg(test)]
mod tests {
use super::*;
use crate::admin::storage_compat::ecstore::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
use crate::admin::storage_compat::lifecycle::tier_last_day_stats::LastDayTierStats;
use http::Uri;
use matchit::Router;
+1 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::admin::router::Operation;
use crate::admin::storage_compat::ecstore::rpc::PeerRestClient;
use crate::admin::storage_compat::PeerRestClient;
use crate::app::context::resolve_endpoints_handle;
use http::StatusCode;
use hyper::Uri;
+25 -27
View File
@@ -14,24 +14,24 @@
use crate::admin::console::{is_console_path, make_console_server};
use crate::admin::handlers::oidc::is_oidc_path;
use crate::admin::storage_compat::ecstore::bucket::bandwidth::monitor::BandwidthDetails;
use crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::{
use crate::admin::storage_compat::GLOBAL_BOOT_TIME;
use crate::admin::storage_compat::PeerRestClient;
use crate::admin::storage_compat::bandwidth::monitor::BandwidthDetails;
use crate::admin::storage_compat::bucket_target_sys::{
BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, TargetClient,
};
use crate::admin::storage_compat::ecstore::bucket::metadata::BUCKET_TARGETS_FILE;
use crate::admin::storage_compat::ecstore::bucket::metadata_sys;
use crate::admin::storage_compat::ecstore::bucket::replication::{
use crate::admin::storage_compat::com::read_config_without_migrate;
use crate::admin::storage_compat::get_global_notification_sys;
use crate::admin::storage_compat::metadata::BUCKET_TARGETS_FILE;
use crate::admin::storage_compat::metadata_sys;
use crate::admin::storage_compat::replication::{
BucketReplicationResyncStatus, BucketStats, GLOBAL_REPLICATION_STATS, ObjectOpts, ReplicationConfigurationExt, ResyncOpts,
get_global_replication_pool,
};
use crate::admin::storage_compat::ecstore::bucket::target::{BucketTarget, BucketTargetType, BucketTargets};
use crate::admin::storage_compat::ecstore::bucket::versioning::VersioningApi;
use crate::admin::storage_compat::ecstore::bucket::versioning_sys::BucketVersioningSys;
use crate::admin::storage_compat::ecstore::config::com::read_config_without_migrate;
use crate::admin::storage_compat::ecstore::global::GLOBAL_BOOT_TIME;
use crate::admin::storage_compat::ecstore::global::{get_global_bucket_monitor, get_global_deployment_id, get_global_region};
use crate::admin::storage_compat::ecstore::notification_sys::get_global_notification_sys;
use crate::admin::storage_compat::ecstore::rpc::PeerRestClient;
use crate::admin::storage_compat::target::{BucketTarget, BucketTargetType, BucketTargets};
use crate::admin::storage_compat::versioning::VersioningApi;
use crate::admin::storage_compat::versioning_sys::BucketVersioningSys;
use crate::admin::storage_compat::{get_global_bucket_monitor, get_global_deployment_id, get_global_region};
use crate::app::context::resolve_object_store_handle;
use crate::app::object_usecase::DefaultObjectUsecase;
use crate::auth::{check_key_valid, get_session_token};
@@ -1420,9 +1420,7 @@ async fn ensure_replication_bucket_exists(bucket: &str) -> S3Result<()> {
async fn ensure_replication_config_exists(bucket: &str) -> S3Result<()> {
match metadata_sys::get_replication_config(bucket).await {
Ok(_) => Ok(()),
Err(crate::admin::storage_compat::ecstore::error::StorageError::ConfigNotFound) => {
Err(s3_error!(ReplicationConfigurationNotFoundError))
}
Err(crate::admin::storage_compat::StorageError::ConfigNotFound) => Err(s3_error!(ReplicationConfigurationNotFoundError)),
Err(err) => Err(ApiError::from(err).into()),
}
}
@@ -1947,7 +1945,7 @@ async fn resolve_replication_target_client(bucket: &str, target: &BucketTarget)
fn build_replication_probe_put_options(now: OffsetDateTime) -> PutObjectOptions {
PutObjectOptions {
internal: crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::AdvancedPutOptions {
internal: crate::admin::storage_compat::bucket_target_sys::AdvancedPutOptions {
source_version_id: Uuid::new_v4().to_string(),
replication_status: ReplicationStatusType::Replica,
source_mtime: now,
@@ -2062,7 +2060,7 @@ async fn source_bucket_requires_object_lock(bucket: &str) -> S3Result<bool> {
.object_lock_enabled
.as_ref()
.is_some_and(|state| state.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)),
Err(crate::admin::storage_compat::ecstore::error::StorageError::ConfigNotFound) => Ok(false),
Err(crate::admin::storage_compat::StorageError::ConfigNotFound) => Ok(false),
Err(err) => Err(ApiError::from(err).into()),
}
}
@@ -2776,7 +2774,7 @@ mod tests {
#[test]
fn apply_replication_reset_to_targets_updates_matching_target() {
let mut targets = BucketTargets {
targets: vec![crate::admin::storage_compat::ecstore::bucket::target::BucketTarget {
targets: vec![crate::admin::storage_compat::target::BucketTarget {
arn: "arn:target".to_string(),
..Default::default()
}],
@@ -2798,10 +2796,10 @@ mod tests {
let mut status = BucketReplicationResyncStatus::new();
status.targets_map.insert(
"arn:z".to_string(),
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus {
crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
resync_id: "rid-z".to_string(),
last_update: Some(datetime!(2025-01-03 00:00 UTC)),
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncFailed,
resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncFailed,
failed_count: 2,
failed_size: 4,
bucket: "bucket-z".to_string(),
@@ -2811,10 +2809,10 @@ mod tests {
);
status.targets_map.insert(
"arn:a".to_string(),
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus {
crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
resync_id: "rid-a".to_string(),
last_update: Some(datetime!(2025-01-02 00:00 UTC)),
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncCompleted,
resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncCompleted,
replicated_count: 3,
replicated_size: 9,
bucket: "bucket-a".to_string(),
@@ -2844,10 +2842,10 @@ mod tests {
let mut status = BucketReplicationResyncStatus::new();
status.targets_map.insert(
"arn:z".to_string(),
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus {
crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
resync_id: "rid-z".to_string(),
last_update: Some(datetime!(2025-02-03 00:00 UTC)),
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncFailed,
resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncFailed,
failed_count: 2,
failed_size: 4,
bucket: "bucket-z".to_string(),
@@ -2857,10 +2855,10 @@ mod tests {
);
status.targets_map.insert(
"arn:a".to_string(),
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus {
crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
resync_id: "rid-a".to_string(),
last_update: Some(datetime!(2025-02-02 00:00 UTC)),
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncCompleted,
resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncCompleted,
replicated_count: 3,
replicated_size: 9,
bucket: "bucket-a".to_string(),
+9 -9
View File
@@ -12,10 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::admin::storage_compat::ecstore::config::com::{STORAGE_CLASS_SUB_SYS, read_config_without_migrate};
use crate::admin::storage_compat::ecstore::config::set_global_storage_class;
use crate::admin::storage_compat::ecstore::config::storageclass;
use crate::admin::storage_compat::ecstore::notification_sys::get_global_notification_sys;
use crate::admin::storage_compat::com::{STORAGE_CLASS_SUB_SYS, read_config_without_migrate};
use crate::admin::storage_compat::get_global_notification_sys;
use crate::admin::storage_compat::set_global_storage_class;
use crate::admin::storage_compat::storageclass;
use crate::app::context::resolve_object_store_handle;
use rustfs_audit::reload_audit_config;
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_WEBHOOK_SUB_SYS};
@@ -371,7 +371,7 @@ pub async fn signal_config_snapshot_reload() {
#[cfg(test)]
mod tests {
use super::*;
use crate::admin::storage_compat::ecstore::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG};
use crate::admin::storage_compat::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG};
use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS;
use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES};
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
@@ -427,7 +427,7 @@ mod tests {
#[test]
fn validate_notify_subsystem_config_rejects_invalid_webhook_endpoint() {
crate::admin::storage_compat::ecstore::config::init();
crate::admin::storage_compat::init();
let mut config = ServerConfig::new();
let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults");
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
@@ -441,7 +441,7 @@ mod tests {
#[test]
fn validate_audit_subsystem_config_rejects_relative_queue_dir() {
crate::admin::storage_compat::ecstore::config::init();
crate::admin::storage_compat::init();
let mut config = ServerConfig::new();
let targets = config.0.get_mut(AUDIT_MQTT_SUB_SYS).expect("audit mqtt defaults");
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
@@ -456,7 +456,7 @@ mod tests {
#[test]
fn validate_identity_openid_config_rejects_missing_openid_scope() {
crate::admin::storage_compat::ecstore::config::init();
crate::admin::storage_compat::init();
let mut config = ServerConfig::new();
let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults");
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
@@ -473,7 +473,7 @@ mod tests {
#[test]
fn validate_identity_openid_config_rejects_invalid_named_provider_id() {
crate::admin::storage_compat::ecstore::config::init();
crate::admin::storage_compat::init();
let mut config = ServerConfig::new();
let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults");
let default_kvs = targets.get(DEFAULT_DELIMITER).cloned().expect("default target");
+2 -2
View File
@@ -13,8 +13,8 @@
// limitations under the License.
use crate::admin::site_replication_identity::{deployment_id_for_endpoint, normalize_peer_map_by_identity_with};
use crate::admin::storage_compat::ecstore::config::com::{read_config, save_config};
use crate::admin::storage_compat::ecstore::error::Error as StorageError;
use crate::admin::storage_compat::Error as StorageError;
use crate::admin::storage_compat::com::{read_config, save_config};
use crate::app::context::resolve_object_store_handle;
use rustfs_madmin::PeerInfo;
use s3s::{S3Error, S3ErrorCode, S3Result};
+33 -75
View File
@@ -12,78 +12,36 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod ecstore {
pub(crate) mod bucket {
pub(crate) use rustfs_ecstore::bucket::{
bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning,
versioning_sys,
};
}
pub(crate) mod client {
pub(crate) use rustfs_ecstore::client::admin_handler_utils;
}
pub(crate) mod config {
pub(crate) use rustfs_ecstore::config::{com, init, set_global_storage_class, storageclass};
}
pub(crate) mod data_usage {
pub(crate) use rustfs_ecstore::data_usage::load_data_usage_from_backend;
}
pub(crate) mod disk {
pub(crate) use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
#[cfg(test)]
pub(crate) use rustfs_ecstore::disk::endpoint;
}
pub(crate) mod endpoints {
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
#[cfg(test)]
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
}
pub(crate) mod error {
pub(crate) use rustfs_ecstore::error::{Error, StorageError};
}
pub(crate) mod global {
pub(crate) use rustfs_ecstore::global::{
GLOBAL_BOOT_TIME, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt, get_global_region,
global_rustfs_port,
};
}
pub(crate) mod metrics_realtime {
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
}
pub(crate) mod notification_sys {
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
}
pub(crate) mod rebalance {
pub(crate) use rustfs_ecstore::rebalance::{
DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::rebalance::{RebalStatus, RebalanceInfo};
}
pub(crate) mod rpc {
pub(crate) use rustfs_ecstore::rpc::PeerRestClient;
}
pub(crate) mod store {
pub(crate) use rustfs_ecstore::store::ECStore;
}
pub(crate) mod store_utils {
pub(crate) use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket;
}
pub(crate) mod tier {
pub(crate) use rustfs_ecstore::tier::{tier, tier_admin, tier_config, tier_handlers};
}
}
pub(crate) use rustfs_ecstore::bucket::lifecycle::tier_last_day_stats::DailyAllTierStats;
pub(crate) use rustfs_ecstore::bucket::{
bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning,
versioning_sys,
};
pub(crate) use rustfs_ecstore::client::admin_handler_utils::AdminError;
pub(crate) use rustfs_ecstore::config::{com, init, set_global_storage_class, storageclass};
pub(crate) use rustfs_ecstore::data_usage::load_data_usage_from_backend;
pub(crate) use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
#[cfg(test)]
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
#[cfg(test)]
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::error::{Error, StorageError};
pub(crate) use rustfs_ecstore::global::{
GLOBAL_BOOT_TIME, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt, get_global_region,
global_rustfs_port,
};
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
pub(crate) use rustfs_ecstore::rebalance::{DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats};
#[cfg(test)]
pub(crate) use rustfs_ecstore::rebalance::{RebalStatus, RebalanceInfo};
pub(crate) use rustfs_ecstore::rpc::PeerRestClient;
pub(crate) use rustfs_ecstore::store::ECStore;
pub(crate) use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket;
pub(crate) use rustfs_ecstore::tier::tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_MISSING_CREDENTIALS};
pub(crate) use rustfs_ecstore::tier::tier_admin::TierCreds;
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierType};
pub(crate) use rustfs_ecstore::tier::tier_handlers::{
ERR_TIER_ALREADY_EXISTS, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND,
};
+6 -8
View File
@@ -15,13 +15,11 @@
//! Admin application use-case contracts.
use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
use crate::app::storage_compat::ecstore::admin_server_info::get_server_info;
use crate::app::storage_compat::ecstore::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend};
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools;
use crate::app::storage_compat::ecstore::pools::{
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
};
use crate::app::storage_compat::ecstore::store::ECStore;
use crate::app::storage_compat::ECStore;
use crate::app::storage_compat::EndpointServerPools;
use crate::app::storage_compat::get_server_info;
use crate::app::storage_compat::{PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free};
use crate::app::storage_compat::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend};
use crate::capacity::resolve_admin_used_capacity;
use crate::error::ApiError;
use crate::server::{DependencyReadiness, collect_dependency_readiness as collect_runtime_dependency_readiness};
@@ -318,7 +316,7 @@ impl DefaultAdminUsecase {
#[cfg(test)]
mod tests {
use super::*;
use crate::app::storage_compat::ecstore::pools::{PoolDecommissionInfo, PoolStatus};
use crate::app::storage_compat::{PoolDecommissionInfo, PoolStatus};
use time::OffsetDateTime;
#[tokio::test]
+7 -9
View File
@@ -20,7 +20,11 @@ use crate::admin::handlers::site_replication::{
use crate::app::context::{
AppContext, default_notify_interface, get_global_app_context, resolve_object_store_handle_for_context,
};
use crate::app::storage_compat::ecstore::bucket::{
use crate::app::storage_compat::ECStore;
use crate::app::storage_compat::StorageError;
use crate::app::storage_compat::get_global_notification_sys;
use crate::app::storage_compat::object_api_utils::to_s3s_etag;
use crate::app::storage_compat::{
bucket_target_sys::BucketTargetSys,
lifecycle::bucket_lifecycle_ops::{
enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, validate_transition_tier,
@@ -38,10 +42,6 @@ use crate::app::storage_compat::ecstore::bucket::{
versioning::VersioningApi,
versioning_sys::BucketVersioningSys,
};
use crate::app::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
use crate::app::storage_compat::ecstore::error::StorageError;
use crate::app::storage_compat::ecstore::notification_sys::get_global_notification_sys;
use crate::app::storage_compat::ecstore::store::ECStore;
use crate::auth::get_condition_values_with_client_info;
use crate::error::ApiError;
use crate::server::RemoteAddr;
@@ -1628,9 +1628,7 @@ impl DefaultBucketUsecase {
}
};
if let Err(err) =
crate::app::storage_compat::ecstore::bucket::lifecycle::lifecycle::Lifecycle::validate(&input_cfg, &rcfg).await
{
if let Err(err) = crate::app::storage_compat::lifecycle::lifecycle::Lifecycle::validate(&input_cfg, &rcfg).await {
return Err(s3_error!(InvalidArgument, "{err}"));
}
@@ -2280,7 +2278,7 @@ mod tests {
BucketTargets {
targets: arns
.iter()
.map(|arn| crate::app::storage_compat::ecstore::bucket::target::BucketTarget {
.map(|arn| crate::app::storage_compat::target::BucketTarget {
arn: (*arn).to_string(),
target_type: BucketTargetType::ReplicationService,
..Default::default()
+2 -7
View File
@@ -12,12 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::app::storage_compat::ecstore::{
bucket::metadata_sys,
disk::endpoint::Endpoint,
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
store::ECStore,
};
use crate::app::storage_compat::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, metadata_sys};
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager};
use rustfs_storage_api::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _};
@@ -82,7 +77,7 @@ async fn setup_capacity_dirty_scope_env() -> (Vec<PathBuf>, Arc<ECStore>) {
};
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
crate::app::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
crate::app::storage_compat::init_local_disks(endpoint_pools.clone())
.await
.unwrap();
+9 -9
View File
@@ -17,11 +17,11 @@ use super::handles::{
default_bucket_metadata_interface, default_endpoints_interface, default_kms_runtime_interface,
default_server_config_interface, default_tier_config_interface,
};
use crate::app::storage_compat::ecstore::bucket::metadata_sys::BucketMetadataSys;
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools;
use crate::app::storage_compat::ecstore::new_object_layer_fn;
use crate::app::storage_compat::ecstore::store::ECStore;
use crate::app::storage_compat::ecstore::tier::tier::TierConfigMgr;
use crate::app::storage_compat::ECStore;
use crate::app::storage_compat::EndpointServerPools;
use crate::app::storage_compat::TierConfigMgr;
use crate::app::storage_compat::metadata_sys::BucketMetadataSys;
use crate::app::storage_compat::new_object_layer_fn;
#[cfg(test)]
use crate::config::RustFSBufferConfig;
use rustfs_config::server_config::Config;
@@ -126,10 +126,10 @@ mod tests {
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
ServerConfigInterface, TierConfigInterface,
};
use crate::app::storage_compat::ecstore::disk::endpoint::Endpoint;
use crate::app::storage_compat::ecstore::endpoints::{Endpoints, PoolEndpoints};
use crate::app::storage_compat::ecstore::new_object_layer_fn;
use crate::app::storage_compat::ecstore::store::init_local_disks;
use crate::app::storage_compat::Endpoint;
use crate::app::storage_compat::init_local_disks;
use crate::app::storage_compat::new_object_layer_fn;
use crate::app::storage_compat::{Endpoints, PoolEndpoints};
use crate::config::{RustFSBufferConfig, WorkloadProfile};
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use std::path::PathBuf;
+1 -1
View File
@@ -21,7 +21,7 @@ use super::interfaces::{
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface,
};
use crate::app::storage_compat::ecstore::{set_object_store_resolver, store::ECStore};
use crate::app::storage_compat::{ECStore, set_object_store_resolver};
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager;
use std::sync::{Arc, OnceLock};
+4 -4
View File
@@ -16,10 +16,10 @@ use super::interfaces::{
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface,
};
use crate::app::storage_compat::ecstore::bucket::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys};
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools;
use crate::app::storage_compat::ecstore::global::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr};
use crate::app::storage_compat::ecstore::tier::tier::TierConfigMgr;
use crate::app::storage_compat::EndpointServerPools;
use crate::app::storage_compat::TierConfigMgr;
use crate::app::storage_compat::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys};
use crate::app::storage_compat::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr};
use crate::config::{RustFSBufferConfig, get_global_buffer_config};
use async_trait::async_trait;
use rustfs_config::server_config::Config;
+3 -3
View File
@@ -12,9 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::app::storage_compat::ecstore::bucket::metadata_sys::BucketMetadataSys;
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools;
use crate::app::storage_compat::ecstore::tier::tier::TierConfigMgr;
use crate::app::storage_compat::EndpointServerPools;
use crate::app::storage_compat::TierConfigMgr;
use crate::app::storage_compat::metadata_sys::BucketMetadataSys;
use crate::config::RustFSBufferConfig;
use async_trait::async_trait;
use rustfs_config::server_config::Config;
+11 -17
View File
@@ -14,19 +14,13 @@
use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase};
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::storage_compat::ecstore::{
bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG},
bucket::metadata_sys,
client::object_api_utils::to_s3s_etag,
client::transition_api::{ReadCloser, ReaderImpl},
disk::endpoint::Endpoint,
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
global::GLOBAL_TierConfigMgr,
store::ECStore,
tier::{
tier_config::{TierConfig, TierType},
warm_backend::{WarmBackend, WarmBackendGetOpts},
},
use crate::app::storage_compat::{
ECStore, Endpoint, EndpointServerPools, Endpoints, GLOBAL_TierConfigMgr, PoolEndpoints, TierConfig, TierType, WarmBackend,
WarmBackendGetOpts,
metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG},
metadata_sys,
object_api_utils::to_s3s_etag,
transition_api::{ReadCloser, ReaderImpl},
};
use crate::storage::ecfs::FS;
use crate::storage::{
@@ -113,7 +107,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
crate::app::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
crate::app::storage_compat::init_local_disks(endpoint_pools.clone())
.await
.unwrap();
@@ -132,7 +126,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
@@ -932,7 +926,7 @@ async fn delete_transitioned_object_removes_remote_tier_copy_via_usecase() {
.expect("Failed to set lifecycle configuration");
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
ecstore.clone(),
bucket.as_str(),
)
@@ -992,7 +986,7 @@ async fn lifecycle_transition_marks_dirty_disks_for_capacity_manager() {
.expect("Failed to set lifecycle configuration");
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
ecstore.clone(),
bucket.as_str(),
)
+36 -56
View File
@@ -16,22 +16,22 @@
use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write};
use crate::app::storage_compat::ecstore::bucket::quota::checker::QuotaChecker;
use crate::app::storage_compat::ecstore::bucket::{
use crate::app::storage_compat::ECStore;
use crate::app::storage_compat::is_disk_compressible;
use crate::app::storage_compat::is_valid_storage_class;
use crate::app::storage_compat::object_api_utils::to_s3s_etag;
use crate::app::storage_compat::quota::checker::QuotaChecker;
#[cfg(test)]
use crate::app::storage_compat::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
use crate::app::storage_compat::{HashReader, WritePlan};
use crate::app::storage_compat::{StorageError, is_err_object_not_found, is_err_version_not_found};
use crate::app::storage_compat::{
lifecycle::{bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::enqueue_transition_immediate},
metadata_sys,
quota::QuotaOperation,
replication::{get_must_replicate_options, must_replicate, schedule_replication},
versioning_sys::BucketVersioningSys,
};
use crate::app::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
use crate::app::storage_compat::ecstore::compress::is_disk_compressible;
use crate::app::storage_compat::ecstore::error::{StorageError, is_err_object_not_found, is_err_version_not_found};
#[cfg(test)]
use crate::app::storage_compat::ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
use crate::app::storage_compat::ecstore::rio::{HashReader, WritePlan};
use crate::app::storage_compat::ecstore::set_disk::is_valid_storage_class;
use crate::app::storage_compat::ecstore::store::ECStore;
use crate::capacity::record_capacity_write;
use crate::error::ApiError;
use crate::storage::access::has_bypass_governance_header;
@@ -479,7 +479,7 @@ impl DefaultMultipartUsecase {
));
}
// Update quota tracking after successful multipart upload
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_write_memory(
crate::app::storage_compat::record_bucket_object_write_memory(
&bucket,
previous_current_size,
obj_info.size.max(0) as u64,
@@ -672,7 +672,7 @@ impl DefaultMultipartUsecase {
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
crate::app::storage_compat::ecstore::rio::compression_metadata_value(CompressionAlgorithm::default()),
crate::app::storage_compat::compression_metadata_value(CompressionAlgorithm::default()),
);
}
@@ -891,18 +891,13 @@ impl DefaultMultipartUsecase {
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
let ssec_write = match ssec_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
ssec_material.key_bytes,
part_id as u32,
)
}
crate::storage::sse::EncryptionKeyKind::Direct => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
ssec_material.key_bytes,
ssec_material.base_nonce,
part_id,
)
crate::app::storage_compat::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
}
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
ssec_material.key_bytes,
ssec_material.base_nonce,
part_id,
),
};
write_plan = write_plan.with_encryption(ssec_write);
(Some(ssec_material.server_side_encryption), ssec_material.kms_key_id)
@@ -918,18 +913,13 @@ impl DefaultMultipartUsecase {
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
let managed_write = match managed_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
managed_material.key_bytes,
part_id as u32,
)
}
crate::storage::sse::EncryptionKeyKind::Direct => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
managed_material.key_bytes,
managed_material.base_nonce,
part_id,
)
crate::app::storage_compat::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
}
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
managed_material.key_bytes,
managed_material.base_nonce,
part_id,
),
};
write_plan = write_plan.with_encryption(managed_write);
(Some(server_side_encryption), ssekms_key_id)
@@ -1244,18 +1234,13 @@ impl DefaultMultipartUsecase {
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
let ssec_write = match ssec_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
ssec_material.key_bytes,
part_id as u32,
)
}
crate::storage::sse::EncryptionKeyKind::Direct => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
ssec_material.key_bytes,
ssec_material.base_nonce,
part_id,
)
crate::app::storage_compat::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
}
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
ssec_material.key_bytes,
ssec_material.base_nonce,
part_id,
),
};
write_plan = write_plan.with_encryption(ssec_write);
(
@@ -1275,18 +1260,13 @@ impl DefaultMultipartUsecase {
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
let managed_write = match managed_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
managed_material.key_bytes,
part_id as u32,
)
}
crate::storage::sse::EncryptionKeyKind::Direct => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
managed_material.key_bytes,
managed_material.base_nonce,
part_id,
)
crate::app::storage_compat::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
}
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
managed_material.key_bytes,
managed_material.base_nonce,
part_id,
),
};
write_plan = write_plan.with_encryption(managed_write);
(Some(server_side_encryption), ssekms_key_id, mp_info.user_defined.clone())
+26 -31
View File
@@ -48,8 +48,18 @@ use metrics::{counter, histogram};
use pin_project_lite::pin_project;
use rustfs_object_capacity::capacity_manager::get_capacity_manager;
// Performance metrics recording (with zero-copy-metrics integration)
use crate::app::storage_compat::ecstore::bucket::quota::checker::QuotaChecker;
use crate::app::storage_compat::ecstore::bucket::{
use crate::app::storage_compat::ECStore;
use crate::app::storage_compat::object_api_utils::to_s3s_etag;
use crate::app::storage_compat::quota::checker::QuotaChecker;
use crate::app::storage_compat::storageclass;
use crate::app::storage_compat::{DiskError, is_all_buckets_not_found};
use crate::app::storage_compat::{DynReader, HashReader, WritePlan, wrap_reader};
use crate::app::storage_compat::{
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
use crate::app::storage_compat::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
use crate::app::storage_compat::{get_lock_acquire_timeout, is_valid_storage_class};
use crate::app::storage_compat::{
lifecycle::{
bucket_lifecycle_audit::LcEventSrc,
bucket_lifecycle_ops::{RestoreRequestOps, enqueue_transition_immediate, post_restore_opts},
@@ -69,16 +79,6 @@ use crate::app::storage_compat::ecstore::bucket::{
versioning::VersioningApi,
versioning_sys::BucketVersioningSys,
};
use crate::app::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
use crate::app::storage_compat::ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
use crate::app::storage_compat::ecstore::config::storageclass;
use crate::app::storage_compat::ecstore::disk::{error::DiskError, error_reduce::is_all_buckets_not_found};
use crate::app::storage_compat::ecstore::error::{
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
use crate::app::storage_compat::ecstore::rio::{DynReader, HashReader, WritePlan, wrap_reader};
use crate::app::storage_compat::ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
use crate::app::storage_compat::ecstore::store::ECStore;
use rustfs_concurrency::GetObjectQueueSnapshot;
use rustfs_filemeta::{
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType,
@@ -276,12 +276,12 @@ async fn enqueue_transitioned_delete_cleanup(
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup);
let je = if opts.delete_prefix {
crate::app::storage_compat::ecstore::bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(
crate::app::storage_compat::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(
&existing.transitioned_object,
)
} else {
let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok());
crate::app::storage_compat::ecstore::bucket::lifecycle::tier_sweeper::transitioned_delete_journal_entry(
crate::app::storage_compat::lifecycle::tier_sweeper::transitioned_delete_journal_entry(
version_id,
opts.versioned,
opts.version_suspended,
@@ -292,10 +292,9 @@ async fn enqueue_transitioned_delete_cleanup(
return Ok(());
};
crate::app::storage_compat::ecstore::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(store, &je)
.await?;
crate::app::storage_compat::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(store, &je).await?;
let mut expiry_state = crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
let mut expiry_state = crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
.write()
.await;
if let Err(err) = expiry_state.enqueue_tier_journal_entry(&je).await {
@@ -1540,7 +1539,7 @@ impl DefaultObjectUsecase {
#[allow(clippy::too_many_arguments)]
async fn prepare_get_object_read(
req: &S3Request<GetObjectInput>,
store: &crate::app::storage_compat::ecstore::store::ECStore,
store: &crate::app::storage_compat::ECStore,
manager: &ConcurrencyManager,
bucket: &str,
key: &str,
@@ -2134,7 +2133,7 @@ impl DefaultObjectUsecase {
insert_str(
&mut metadata,
SUFFIX_COMPRESSION,
crate::app::storage_compat::ecstore::rio::compression_metadata_value(algorithm),
crate::app::storage_compat::compression_metadata_value(algorithm),
);
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
@@ -2149,7 +2148,7 @@ impl DefaultObjectUsecase {
insert_str(
&mut opts.user_defined,
SUFFIX_COMPRESSION,
crate::app::storage_compat::ecstore::rio::compression_metadata_value(algorithm),
crate::app::storage_compat::compression_metadata_value(algorithm),
);
insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string());
@@ -2341,7 +2340,7 @@ impl DefaultObjectUsecase {
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
// Fast in-memory update for immediate quota and admin usage consistency
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_write_memory(
crate::app::storage_compat::record_bucket_object_write_memory(
&bucket,
previous_current_size,
obj_info.size.max(0) as u64,
@@ -3211,7 +3210,7 @@ impl DefaultObjectUsecase {
insert_str(
&mut compress_metadata,
SUFFIX_COMPRESSION,
crate::app::storage_compat::ecstore::rio::compression_metadata_value(CompressionAlgorithm::default()),
crate::app::storage_compat::compression_metadata_value(CompressionAlgorithm::default()),
);
insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
} else {
@@ -3304,12 +3303,8 @@ impl DefaultObjectUsecase {
// Update quota tracking after successful copy
if has_bucket_metadata {
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_write_memory(
&bucket,
previous_current_size,
oi.size.max(0) as u64,
)
.await;
crate::app::storage_compat::record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64)
.await;
}
let raw_dest_version = oi.version_id.map(|v| v.to_string());
@@ -3586,7 +3581,7 @@ impl DefaultObjectUsecase {
);
}
let size = object_sizes[i].max(0) as u64;
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_delete_memory(
crate::app::storage_compat::record_bucket_object_delete_memory(
&bucket,
size,
existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
@@ -3824,7 +3819,7 @@ impl DefaultObjectUsecase {
}
// Fast in-memory update for immediate quota and admin usage consistency
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_delete_memory(
crate::app::storage_compat::record_bucket_object_delete_memory(
&bucket,
obj_info.size.max(0) as u64,
opts.version_id.is_none(),
@@ -4757,7 +4752,7 @@ impl DefaultObjectUsecase {
insert_str(
&mut metadata,
SUFFIX_COMPRESSION,
crate::app::storage_compat::ecstore::rio::compression_metadata_value(algorithm),
crate::app::storage_compat::compression_metadata_value(algorithm),
);
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
+47 -93
View File
@@ -12,96 +12,50 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod ecstore {
pub(crate) use rustfs_ecstore::global::{new_object_layer_fn, set_object_store_resolver};
pub(crate) mod admin_server_info {
pub(crate) use rustfs_ecstore::admin_server_info::get_server_info;
}
pub(crate) mod bucket {
pub(crate) use rustfs_ecstore::bucket::{
bucket_target_sys, lifecycle, metadata, metadata_sys, object_lock, policy_sys, quota, replication, tagging, target,
utils, versioning, versioning_sys,
};
}
pub(crate) mod client {
pub(crate) use rustfs_ecstore::client::object_api_utils;
#[cfg(test)]
pub(crate) use rustfs_ecstore::client::transition_api;
}
pub(crate) mod compress {
pub(crate) use rustfs_ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
}
pub(crate) mod config {
pub(crate) use rustfs_ecstore::config::storageclass;
}
pub(crate) mod data_usage {
pub(crate) use rustfs_ecstore::data_usage::{
apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory,
record_bucket_object_write_memory,
};
}
pub(crate) mod disk {
#[cfg(test)]
pub(crate) use rustfs_ecstore::disk::endpoint;
pub(crate) use rustfs_ecstore::disk::{error, error_reduce};
}
pub(crate) mod endpoints {
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
#[cfg(test)]
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
}
pub(crate) mod error {
pub(crate) use rustfs_ecstore::error::{
Error, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
}
pub(crate) mod global {
#[cfg(test)]
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
pub(crate) use rustfs_ecstore::global::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr};
}
pub(crate) mod notification_sys {
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
}
pub(crate) mod pools {
pub(crate) use rustfs_ecstore::pools::{
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
};
}
pub(crate) mod rio {
#[cfg(test)]
pub(crate) use rustfs_ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader};
pub(crate) use rustfs_ecstore::rio::{
DynReader, HashReader, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader,
};
}
pub(crate) mod set_disk {
pub(crate) use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
}
pub(crate) mod store {
pub(crate) use rustfs_ecstore::store::ECStore;
#[cfg(test)]
pub(crate) use rustfs_ecstore::store::init_local_disks;
}
pub(crate) mod tier {
pub(crate) use rustfs_ecstore::tier::tier;
#[cfg(test)]
pub(crate) use rustfs_ecstore::tier::{tier_config, warm_backend};
}
}
pub(crate) use rustfs_ecstore::admin_server_info::get_server_info;
pub(crate) use rustfs_ecstore::bucket::{
bucket_target_sys, lifecycle, metadata, metadata_sys, object_lock, policy_sys, quota, replication, tagging, target, utils,
versioning, versioning_sys,
};
pub(crate) use rustfs_ecstore::client::object_api_utils;
#[cfg(test)]
pub(crate) use rustfs_ecstore::client::transition_api;
pub(crate) use rustfs_ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
pub(crate) use rustfs_ecstore::config::storageclass;
pub(crate) use rustfs_ecstore::data_usage::{
apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory,
record_bucket_object_write_memory,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
pub(crate) use rustfs_ecstore::disk::error::DiskError;
pub(crate) use rustfs_ecstore::disk::error_reduce::is_all_buckets_not_found;
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
#[cfg(test)]
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::error::{
Error, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
pub(crate) use rustfs_ecstore::global::{
get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr, new_object_layer_fn, set_object_store_resolver,
};
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
pub(crate) use rustfs_ecstore::pools::{
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader};
pub(crate) use rustfs_ecstore::rio::{
DynReader, HashReader, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader,
};
pub(crate) use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
pub(crate) use rustfs_ecstore::store::ECStore;
#[cfg(test)]
pub(crate) use rustfs_ecstore::store::init_local_disks;
pub(crate) use rustfs_ecstore::tier::tier::TierConfigMgr;
#[cfg(test)]
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierType};
#[cfg(test)]
pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, WarmBackendGetOpts};
+2 -2
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_compat::ecstore::disk::DiskAPI;
use crate::storage_compat::DiskAPI;
use rustfs_io_metrics::capacity_metrics::{
record_capacity_cache_hit, record_capacity_cache_miss, record_capacity_cache_served, record_capacity_refresh_request,
record_capacity_scan_mode,
@@ -263,7 +263,7 @@ pub async fn init_capacity_management_for_local_disks() {
"Capacity manager state changed"
);
let disks = crate::storage_compat::ecstore::store::all_local_disk().await;
let disks = crate::storage_compat::all_local_disk().await;
if disks.is_empty() {
warn!(
component = LOG_COMPONENT_CAPACITY,
+2 -2
View File
@@ -16,7 +16,7 @@
#[allow(unsafe_op_in_unsafe_fn)]
mod tests {
use crate::config::{CommandResult, Config, Opt, TlsCommands};
use crate::storage_compat::ecstore::disks_layout::DisksLayout;
use crate::storage_compat::DisksLayout;
use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_OBS_ENDPOINT, RUSTFS_REGION};
use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY};
use serial_test::serial;
@@ -262,7 +262,7 @@ mod tests {
#[test]
#[serial]
fn test_volumes_and_disk_layout_parsing() {
use crate::storage_compat::ecstore::disks_layout::DisksLayout;
use crate::storage_compat::DisksLayout;
// Test case 1: Single volume path
let args = vec!["rustfs", "/data/vol1"];
+9 -18
View File
@@ -51,20 +51,11 @@ use crate::init::{add_bucket_notification_configuration, init_buffer_profile_sys
use crate::server::{ShutdownHandle, shutdown_event_notifier, start_http_server, stop_audit_system};
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
use crate::startup_iam::{bootstrap_or_defer_iam_init, publish_ready_for_iam_bootstrap};
use crate::storage_compat::ecstore::store::init_lock_clients;
use crate::storage_compat::ecstore::{
bucket::replication::init_background_replication,
bucket::{
metadata_sys::init_bucket_metadata_sys,
migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
},
config as ecconfig,
endpoints::EndpointServerPools,
global::set_global_rustfs_port,
set_global_endpoints,
store::ECStore,
store::init_local_disks,
update_erasure_type,
use crate::storage_compat::init_lock_clients;
use crate::storage_compat::{
ECStore, EndpointServerPools, init as init_ecstore_config, init_background_replication, init_bucket_metadata_sys,
init_global_config_sys, init_local_disks, set_global_endpoints, set_global_rustfs_port, try_migrate_bucket_metadata,
try_migrate_iam_config, try_migrate_server_config, update_erasure_type,
};
use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
use rustfs_credentials::init_global_action_credentials;
@@ -333,7 +324,7 @@ impl RustFSServerBuilder {
let region = region_str
.parse()
.map_err(|e| ServerError::Init(format!("invalid region '{region_str}': {e}")))?;
crate::storage_compat::ecstore::global::set_global_region(region);
crate::storage_compat::set_global_region(region);
}
let server_port = server_addr.port();
@@ -389,12 +380,12 @@ impl RustFSServerBuilder {
}
};
ecconfig::init();
ecconfig::try_migrate_server_config(store.clone()).await;
init_ecstore_config();
try_migrate_server_config(store.clone()).await;
// Global config system (with retry).
let mut retry = 0;
while let Err(e) = ecconfig::init_global_config_sys(store.clone()).await {
while let Err(e) = init_global_config_sys(store.clone()).await {
retry += 1;
if retry > 15 {
shutdown_embedded_server();
+2 -2
View File
@@ -12,8 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_compat::ecstore::bucket::quota::QuotaError;
use crate::storage_compat::ecstore::error::StorageError;
use crate::storage_compat::StorageError;
use crate::storage_compat::quota::QuotaError;
use rustfs_storage_api::HTTPRangeError;
use s3s::{S3Error, S3ErrorCode};
+2 -2
View File
@@ -14,7 +14,7 @@
use crate::server::ShutdownHandle;
use crate::storage::{process_lambda_configurations, process_queue_configurations, process_topic_configurations};
use crate::storage_compat::ecstore::bucket::metadata_sys;
use crate::storage_compat::metadata_sys;
use crate::{admin, config, version};
use rustfs_config::{
DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK,
@@ -157,7 +157,7 @@ fn arn_to_target_id(arn_str: &str) -> Result<rustfs_targets::arn::TargetID, Targ
/// * `buckets` - A vector of bucket names to process
#[instrument(skip_all)]
pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
let global_region = crate::storage_compat::ecstore::global::get_global_region();
let global_region = crate::storage_compat::get_global_region();
let region = global_region
.as_ref()
.filter(|r| !r.as_str().is_empty())
+1 -1
View File
@@ -14,7 +14,7 @@
use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store};
use crate::app::context::resolve_server_config;
use crate::storage_compat::ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook};
use crate::storage_compat::{EcstoreEventArgs, register_event_dispatch_hook};
use rustfs_notify::EventArgs as NotifyEventArgs;
use rustfs_s3_types::EventName;
use std::net::SocketAddr;
+1 -1
View File
@@ -33,7 +33,7 @@ use crate::server::{
use crate::storage;
use crate::storage::rpc::InternodeRpcService;
use crate::storage::tonic_service::make_server;
use crate::storage_compat::ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
use crate::storage_compat::{TONIC_RPC_PREFIX, verify_rpc_signature};
use bytes::Bytes;
use http::{HeaderMap, Method, Request as HttpRequest, Response};
use hyper_util::{
+3 -3
View File
@@ -12,9 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_compat::ecstore::{
config::com::{read_config, save_config},
error::Error as StorageError,
use crate::storage_compat::{
EcstoreError as StorageError,
com::{read_config, save_config},
resolve_object_store_handle,
};
use serde::{Deserialize, Serialize};
+10 -13
View File
@@ -14,9 +14,9 @@
use crate::server::{ServiceState, ServiceStateManager};
use crate::server::{has_path_prefix, is_table_catalog_path};
use crate::storage_compat::ecstore::global::is_dist_erasure;
use crate::storage_compat::ecstore::global::{get_global_endpoints_opt, get_global_lock_clients};
use crate::storage_compat::ecstore::resolve_object_store_handle;
use crate::storage_compat::is_dist_erasure;
use crate::storage_compat::resolve_object_store_handle;
use crate::storage_compat::{get_global_endpoints_opt, get_global_lock_clients};
use bytes::Bytes;
use http::{Request as HttpRequest, Response, StatusCode};
use http_body::Body;
@@ -496,13 +496,10 @@ async fn collect_storage_readiness_uncached() -> bool {
}
}
fn set_lock_quorum_status(
online_hosts: &HashSet<String>,
set_endpoints: &[crate::storage_compat::ecstore::disk::endpoint::Endpoint],
) -> LockQuorumStatus {
fn set_lock_quorum_status(online_hosts: &HashSet<String>, set_endpoints: &[crate::storage_compat::Endpoint]) -> LockQuorumStatus {
let total_clients = set_endpoints
.iter()
.map(crate::storage_compat::ecstore::disk::endpoint::Endpoint::host_port)
.map(crate::storage_compat::Endpoint::host_port)
.filter(|host| !host.is_empty())
.collect::<HashSet<_>>();
let total_clients_len = total_clients.len();
@@ -526,7 +523,7 @@ fn set_lock_quorum_status(
}
fn aggregate_lock_quorum_status(
pool_endpoints: &crate::storage_compat::ecstore::endpoints::EndpointServerPools,
pool_endpoints: &crate::storage_compat::EndpointServerPools,
online_hosts: &HashSet<String>,
) -> LockQuorumStatus {
let mut connected_clients = 0usize;
@@ -842,8 +839,8 @@ mod tests {
#[test]
fn aggregate_lock_quorum_status_requires_each_set_to_meet_quorum() {
use crate::storage_compat::ecstore::disk::endpoint::Endpoint;
use crate::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::storage_compat::Endpoint;
use crate::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints};
let endpoints = vec![
Endpoint {
@@ -900,8 +897,8 @@ mod tests {
#[test]
fn aggregate_lock_quorum_status_fails_when_any_set_loses_quorum() {
use crate::storage_compat::ecstore::disk::endpoint::Endpoint;
use crate::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use crate::storage_compat::Endpoint;
use crate::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints};
let endpoints = vec![
Endpoint {
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_compat::ecstore::endpoints::EndpointServerPools;
use crate::storage_compat::EndpointServerPools;
use rustfs_config::{
DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, ENV_RUSTFS_UNSUPPORTED_FS_POLICY, RUSTFS_UNSUPPORTED_FS_POLICY_FAIL,
RUSTFS_UNSUPPORTED_FS_POLICY_WARN,
+1 -1
View File
@@ -14,7 +14,7 @@
use crate::app::context::{AppContext, get_global_app_context, init_global_app_context};
use crate::server::{ServiceStateManager, publish_ready_when_runtime_ready};
use crate::storage_compat::ecstore::store::ECStore;
use crate::storage_compat::ECStore;
use rustfs_common::{GlobalReadiness, SystemStage};
use rustfs_iam::init_iam_sys;
use rustfs_kms::KmsServiceManager;
+2 -2
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_compat::ecstore::global::set_global_rustfs_port;
use crate::storage_compat::set_global_rustfs_port;
use crate::{
capacity::capacity_integration::init_capacity_management,
config::Config,
@@ -57,7 +57,7 @@ pub async fn init_startup_listen_context(config: &Config) -> Result<StartupListe
if let Some(region_str) = &config.region {
region_str
.parse::<s3s::region::Region>()
.map(crate::storage_compat::ecstore::global::set_global_region)
.map(crate::storage_compat::set_global_region)
.map_err(|err| Error::other(format!("invalid region '{}': {}", region_str, err)))?;
}
+7 -17
View File
@@ -12,16 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_compat::ecstore::{
bucket::{
metadata_sys::init_bucket_metadata_sys,
migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
replication::get_global_replication_pool,
},
endpoints::EndpointServerPools,
global::shutdown_background_services,
notification_sys::new_global_notification_sys,
store::ECStore,
use crate::storage_compat::{
ECStore, EndpointServerPools, get_global_replication_pool, init_bucket_metadata_sys, new_global_notification_sys,
shutdown_background_services, try_migrate_bucket_metadata, try_migrate_iam_config,
};
use crate::{
config::Config,
@@ -597,16 +590,14 @@ where
start_audit().await
}
pub async fn init_notification_system(endpoint_pools: EndpointServerPools) -> crate::storage_compat::ecstore::error::Result<()> {
pub async fn init_notification_system(endpoint_pools: EndpointServerPools) -> crate::storage_compat::EcstoreResult<()> {
init_notification_system_with(|| new_global_notification_sys(endpoint_pools)).await
}
async fn init_notification_system_with<InitFn, InitFuture>(
init_notification: InitFn,
) -> crate::storage_compat::ecstore::error::Result<()>
async fn init_notification_system_with<InitFn, InitFuture>(init_notification: InitFn) -> crate::storage_compat::EcstoreResult<()>
where
InitFn: FnOnce() -> InitFuture,
InitFuture: Future<Output = crate::storage_compat::ecstore::error::Result<()>>,
InitFuture: Future<Output = crate::storage_compat::EcstoreResult<()>>,
{
init_notification().await
}
@@ -664,8 +655,7 @@ mod tests {
#[tokio::test]
async fn notification_system_returns_source_error() {
let result =
init_notification_system_with(|| async { Err(crate::storage_compat::ecstore::error::Error::FaultyDisk) }).await;
let result = init_notification_system_with(|| async { Err(crate::storage_compat::EcstoreError::FaultyDisk) }).await;
assert!(result.is_err());
}
+6 -9
View File
@@ -13,12 +13,9 @@
// limitations under the License.
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
use crate::storage_compat::ecstore::{
bucket::replication::init_background_replication,
config as ecconfig,
endpoints::EndpointServerPools,
set_global_endpoints,
store::{ECStore, init_local_disks, init_lock_clients, prewarm_local_disk_id_map},
use crate::storage_compat::{
ECStore, EndpointServerPools, init as init_ecstore_config, init_background_replication, init_global_config_sys,
init_local_disks, init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints, try_migrate_server_config,
update_erasure_type,
};
use rustfs_common::{GlobalReadiness, SystemStage};
@@ -147,11 +144,11 @@ pub async fn init_startup_storage_runtime(
}
async fn init_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> {
ecconfig::init();
ecconfig::try_migrate_server_config(store.clone()).await;
init_ecstore_config();
try_migrate_server_config(store.clone()).await;
let mut retry_count = 0;
while let Err(e) = ecconfig::init_global_config_sys(store.clone()).await {
while let Err(e) = init_global_config_sys(store.clone()).await {
let next_retry_count = retry_count + 1;
error!(
target: "rustfs::main::run",
+6 -6
View File
@@ -18,11 +18,11 @@ use crate::error::ApiError;
use crate::license::license_check;
use crate::server::RemoteAddr;
use crate::storage::request_context::RequestContext;
use crate::storage::storage_compat::ecstore::bucket::metadata_sys;
use crate::storage::storage_compat::ecstore::bucket::policy_sys::PolicySys;
use crate::storage::storage_compat::ecstore::error::{StorageError, is_err_bucket_not_found};
use crate::storage::storage_compat::ecstore::resolve_object_store_handle;
use crate::storage::storage_compat::ecstore::store::ECStore;
use crate::storage::storage_compat::ECStore;
use crate::storage::storage_compat::metadata_sys;
use crate::storage::storage_compat::policy_sys::PolicySys;
use crate::storage::storage_compat::resolve_object_store_handle;
use crate::storage::storage_compat::{StorageError, is_err_bucket_not_found};
use metrics::counter;
use rustfs_iam::error::Error as IamError;
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
@@ -930,7 +930,7 @@ impl S3Access for FS {
let req_info = ReqInfo {
cred,
is_owner,
region: crate::storage::storage_compat::ecstore::global::get_global_region(),
region: crate::storage::storage_compat::get_global_region(),
request_context,
..Default::default()
};
+12 -14
View File
@@ -21,21 +21,19 @@ use crate::storage::access::has_bypass_governance_header;
use crate::storage::helper::OperationHelper;
use crate::storage::options::get_opts;
use crate::storage::s3_api::acl;
use crate::storage::storage_compat::ecstore::{
bucket::{
metadata::{
BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG,
BUCKET_WEBSITE_CONFIG, OBJECT_LOCK_CONFIG,
},
metadata_sys,
object_lock::objectlock_sys::check_retention_for_modification,
replication::{GLOBAL_REPLICATION_STATS, ReplicationConfigurationExt},
tagging::{decode_tags, decode_tags_to_map, encode_tags},
utils::serialize,
versioning::VersioningApi,
versioning_sys::BucketVersioningSys,
use crate::storage::storage_compat::{
StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
metadata::{
BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG,
BUCKET_WEBSITE_CONFIG, OBJECT_LOCK_CONFIG,
},
error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found},
metadata_sys,
object_lock::objectlock_sys::check_retention_for_modification,
replication::{GLOBAL_REPLICATION_STATS, ReplicationConfigurationExt},
tagging::{decode_tags, decode_tags_to_map, encode_tags},
utils::serialize,
versioning::VersioningApi,
versioning_sys::BucketVersioningSys,
};
use crate::storage::{parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled};
use crate::table_catalog;
+7 -7
View File
@@ -16,12 +16,12 @@ use crate::config::{RustFSBufferConfig, WorkloadProfile, get_global_buffer_confi
use crate::error::ApiError;
use crate::server::cors;
use crate::storage::ecfs::ListObjectUnorderedQuery;
use crate::storage::storage_compat::ecstore::bucket::metadata_sys;
use crate::storage::storage_compat::ecstore::bucket::metadata_sys::get_replication_config;
use crate::storage::storage_compat::ecstore::bucket::object_lock::objectlock_sys;
use crate::storage::storage_compat::ecstore::bucket::replication::ReplicationConfigurationExt;
use crate::storage::storage_compat::ecstore::error::StorageError;
use crate::storage::storage_compat::ecstore::resolve_object_store_handle;
use crate::storage::storage_compat::StorageError;
use crate::storage::storage_compat::metadata_sys;
use crate::storage::storage_compat::metadata_sys::get_replication_config;
use crate::storage::storage_compat::object_lock::objectlock_sys;
use crate::storage::storage_compat::replication::ReplicationConfigurationExt;
use crate::storage::storage_compat::resolve_object_store_handle;
use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE};
use http::{HeaderMap, HeaderValue, StatusCode};
use metrics::counter;
@@ -739,7 +739,7 @@ pub(crate) async fn has_replication_rules(bucket: &str, objects: &[ObjectToDelet
}
/// Helper function to get store and validate bucket exists
pub(crate) async fn get_validated_store(bucket: &str) -> S3Result<Arc<crate::storage::storage_compat::ecstore::store::ECStore>> {
pub(crate) async fn get_validated_store(bucket: &str) -> S3Result<Arc<crate::storage::storage_compat::ECStore>> {
let Some(store) = resolve_object_store_handle() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
+6 -6
View File
@@ -18,8 +18,8 @@ mod tests {
use crate::server::cors;
use crate::storage::ecfs::{FS, validate_object_lock_configuration_input};
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
use crate::storage::storage_compat::ecstore::bucket::{metadata::BucketMetadata, metadata_sys};
use crate::storage::storage_compat::ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
use crate::storage::storage_compat::DEFAULT_READ_BUFFER_SIZE;
use crate::storage::storage_compat::{metadata::BucketMetadata, metadata_sys};
use crate::storage::{
StorageObjectInfo as ObjectInfo, apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions,
get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal, matches_origin_pattern, parse_etag,
@@ -940,12 +940,12 @@ mod tests {
#[tokio::test]
async fn test_validate_bucket_object_lock_enabled() {
use crate::storage::storage_compat::ecstore::bucket::metadata::BucketMetadata;
use crate::storage::storage_compat::ecstore::bucket::metadata_sys::set_bucket_metadata;
use crate::storage::storage_compat::metadata::BucketMetadata;
use crate::storage::storage_compat::metadata_sys::set_bucket_metadata;
use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
use time::OffsetDateTime;
if crate::storage::storage_compat::ecstore::bucket::metadata_sys::GLOBAL_BucketMetadataSys
if crate::storage::storage_compat::metadata_sys::GLOBAL_BucketMetadataSys
.get()
.is_none()
{
@@ -1783,7 +1783,7 @@ mod tests {
/// with a single-element vec value, matching the format expected by policy evaluation.
#[test]
fn test_object_tag_condition_key_format() {
use crate::storage::storage_compat::ecstore::bucket::tagging::decode_tags_to_map;
use crate::storage::storage_compat::tagging::decode_tags_to_map;
use std::collections::HashMap;
let tags_str = "security=public&project=webapp&env=prod";
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage::storage_compat::ecstore::store::ECStore;
use crate::storage::storage_compat::ECStore;
use rustfs_storage_api::ListOperations as _;
use std::sync::Arc;
+4 -4
View File
@@ -29,11 +29,11 @@ pub mod timeout_wrapper;
pub mod tonic_service;
pub(crate) type StorageDeletedObject = rustfs_storage_api::DeletedObject;
pub(crate) type StorageGetObjectReader = crate::storage::storage_compat::ecstore::GetObjectReader;
pub(crate) type StorageObjectInfo = crate::storage::storage_compat::ecstore::ObjectInfo;
pub(crate) type StorageObjectOptions = crate::storage::storage_compat::ecstore::ObjectOptions;
pub(crate) type StorageGetObjectReader = crate::storage::storage_compat::GetObjectReader;
pub(crate) type StorageObjectInfo = crate::storage::storage_compat::ObjectInfo;
pub(crate) type StorageObjectOptions = crate::storage::storage_compat::ObjectOptions;
pub(crate) type StorageObjectToDelete = rustfs_storage_api::ObjectToDelete;
pub(crate) type StoragePutObjReader = crate::storage::storage_compat::ecstore::PutObjReader;
pub(crate) type StoragePutObjReader = crate::storage::storage_compat::PutObjReader;
#[cfg(test)]
mod concurrent_fix_test;
+3 -3
View File
@@ -12,9 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage::storage_compat::ecstore::bucket::versioning_sys::BucketVersioningSys;
use crate::storage::storage_compat::ecstore::error::Result;
use crate::storage::storage_compat::ecstore::error::StorageError;
use crate::storage::storage_compat::Result;
use crate::storage::storage_compat::StorageError;
use crate::storage::storage_compat::versioning_sys::BucketVersioningSys;
use http::header::{IF_MATCH, IF_NONE_MATCH};
use http::{HeaderMap, HeaderValue};
use rustfs_utils::http::{
+4 -4
View File
@@ -14,10 +14,10 @@
use crate::server::RPC_PREFIX;
use crate::storage::request_context::spawn_traced;
use crate::storage::storage_compat::ecstore::disk::{DiskAPI, WalkDirOptions};
use crate::storage::storage_compat::ecstore::rpc::verify_rpc_signature;
use crate::storage::storage_compat::ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
use crate::storage::storage_compat::ecstore::store::find_local_disk_by_ref;
use crate::storage::storage_compat::DEFAULT_READ_BUFFER_SIZE;
use crate::storage::storage_compat::find_local_disk_by_ref;
use crate::storage::storage_compat::verify_rpc_signature;
use crate::storage::storage_compat::{DiskAPI, WalkDirOptions};
use bytes::{Bytes, BytesMut};
use futures_util::TryStreamExt;
use http::{HeaderMap, Method, Request, Response, StatusCode, Uri};
+10 -23
View File
@@ -16,22 +16,12 @@ use crate::admin::service::{
config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot},
site_replication::reload_site_replication_runtime_state,
};
use crate::storage::storage_compat::ecstore::{
admin_server_info::get_local_server_property,
bucket::{metadata::load_bucket_metadata, metadata_sys},
disk::{
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
UpdateMetadataOpts, error::DiskError,
},
get_global_lock_client,
global::GLOBAL_TierConfigMgr,
metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics},
resolve_object_store_handle,
rpc::{
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC,
},
store::{all_local_disk_path, find_local_disk_by_ref},
use crate::storage::storage_compat::{
CollectMetricsOpts, DeleteOptions, DiskAPI, DiskError, DiskInfoOptions, DiskStore, FileInfoVersions, GLOBAL_TierConfigMgr,
LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, ReadMultipleReq, ReadMultipleResp,
ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, UpdateMetadataOpts, all_local_disk_path,
collect_local_metrics, find_local_disk_by_ref, get_global_lock_client, get_local_server_property,
metadata::load_bucket_metadata, metadata_sys, resolve_object_store_handle,
};
use bytes::Bytes;
use futures::Stream;
@@ -132,9 +122,7 @@ fn unimplemented_rpc(method: &str) -> Status {
Status::unimplemented(format!("{method} is not implemented"))
}
fn background_rebalance_start_error_message(
result: crate::storage::storage_compat::ecstore::error::Result<()>,
) -> Option<String> {
fn background_rebalance_start_error_message(result: crate::storage::storage_compat::Result<()>) -> Option<String> {
result.err().map(|err| format!("start_rebalance failed: {err}"))
}
@@ -2378,9 +2366,8 @@ mod tests {
#[test]
fn test_background_rebalance_start_error_message_formats_error() {
let message =
background_rebalance_start_error_message(Err(crate::storage::storage_compat::ecstore::error::Error::other("boom")))
.expect("background rebalance start failure should be formatted");
let message = background_rebalance_start_error_message(Err(crate::storage::storage_compat::Error::other("boom")))
.expect("background rebalance start failure should be formatted");
assert!(message.contains("start_rebalance failed"));
assert!(message.contains("boom"));
@@ -2710,7 +2697,7 @@ mod tests {
vars.insert(PEER_RESTSIGNAL.to_string(), SERVICE_SIGNAL_RELOAD_DYNAMIC.to_string());
vars.insert(
PEER_RESTSUB_SYS.to_string(),
crate::storage::storage_compat::ecstore::config::com::STORAGE_CLASS_SUB_SYS.to_string(),
crate::storage::storage_compat::com::STORAGE_CLASS_SUB_SYS.to_string(),
);
let request = Request::new(SignalServiceRequest {
+1 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::storage::s3_api::common::rustfs_owner;
use crate::storage::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
use crate::storage::storage_compat::object_api_utils::to_s3s_etag;
use percent_encoding::percent_decode_str;
use rustfs_storage_api::{
BucketInfo, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
+2 -2
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
use crate::storage::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
use crate::storage::storage_compat::object_api_utils::to_s3s_etag;
use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo};
use s3s::dto::{CommonPrefix, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp};
use s3s::{S3Error, S3ErrorCode};
@@ -191,7 +191,7 @@ mod tests {
parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
};
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
use crate::storage::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
use crate::storage::storage_compat::object_api_utils::to_s3s_etag;
use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo};
use s3s::S3ErrorCode;
use s3s::dto::Timestamp;
+8 -18
View File
@@ -69,7 +69,7 @@
//! }
//! ```
use crate::storage::storage_compat::ecstore::error::StorageError;
use crate::storage::storage_compat::StorageError;
use aes_gcm::{
Aes256Gcm, Key, Nonce,
aead::{Aead, KeyInit},
@@ -128,8 +128,8 @@ const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE;
const OBJECT_KEY_DERIVATION_CONTEXT: &[u8] = b"object-encryption-key generation";
use crate::error::ApiError;
use crate::storage::storage_compat::ecstore::bucket::metadata_sys;
use crate::storage::storage_compat::ecstore::error::Error;
use crate::storage::storage_compat::Error;
use crate::storage::storage_compat::metadata_sys;
use rustfs_utils::http::headers::{
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT,
@@ -745,29 +745,19 @@ pub struct ManagedSealedKey {
}
impl EncryptionMaterial {
pub fn write_encryption(
&self,
multipart_part_number: Option<usize>,
) -> crate::storage::storage_compat::ecstore::rio::WriteEncryption {
pub fn write_encryption(&self, multipart_part_number: Option<usize>) -> crate::storage::storage_compat::WriteEncryption {
match (self.key_kind, multipart_part_number) {
(EncryptionKeyKind::Object, Some(part_number)) => {
crate::storage::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
self.key_bytes,
part_number as u32,
)
crate::storage::storage_compat::WriteEncryption::multipart_object_key(self.key_bytes, part_number as u32)
}
(EncryptionKeyKind::Object, None) => {
crate::storage::storage_compat::ecstore::rio::WriteEncryption::singlepart_object_key(self.key_bytes)
crate::storage::storage_compat::WriteEncryption::singlepart_object_key(self.key_bytes)
}
(EncryptionKeyKind::Direct, Some(part_number)) => {
crate::storage::storage_compat::ecstore::rio::WriteEncryption::multipart(
self.key_bytes,
self.base_nonce,
part_number,
)
crate::storage::storage_compat::WriteEncryption::multipart(self.key_bytes, self.base_nonce, part_number)
}
(EncryptionKeyKind::Direct, None) => {
crate::storage::storage_compat::ecstore::rio::WriteEncryption::singlepart(self.key_bytes, self.base_nonce)
crate::storage::storage_compat::WriteEncryption::singlepart(self.key_bytes, self.base_nonce)
}
}
}
+30 -66
View File
@@ -12,70 +12,34 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod ecstore {
pub(crate) use rustfs_ecstore::global::{get_global_lock_client, resolve_object_store_handle};
pub(crate) use rustfs_ecstore::admin_server_info::get_local_server_property;
pub(crate) use rustfs_ecstore::bucket::{
metadata, metadata_sys, object_lock, policy_sys, replication, tagging, utils, versioning, versioning_sys,
};
pub(crate) use rustfs_ecstore::client::object_api_utils;
#[cfg(test)]
pub(crate) use rustfs_ecstore::config::com;
pub(crate) use rustfs_ecstore::disk::error::DiskError;
pub(crate) use rustfs_ecstore::disk::{
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
UpdateMetadataOpts, WalkDirOptions,
};
pub(crate) use rustfs_ecstore::error::{
Error, Result, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
pub(crate) use rustfs_ecstore::global::{
GLOBAL_TierConfigMgr, get_global_lock_client, get_global_region, resolve_object_store_handle,
};
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
pub(crate) use rustfs_ecstore::rio::WriteEncryption;
pub(crate) use rustfs_ecstore::rpc::{
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, verify_rpc_signature,
};
pub(crate) use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk_path, find_local_disk_by_ref};
pub(crate) mod admin_server_info {
pub(crate) use rustfs_ecstore::admin_server_info::get_local_server_property;
}
pub(crate) mod bucket {
pub(crate) use rustfs_ecstore::bucket::{
metadata, metadata_sys, object_lock, policy_sys, replication, tagging, utils, versioning, versioning_sys,
};
}
pub(crate) mod client {
pub(crate) use rustfs_ecstore::client::object_api_utils;
}
#[cfg(test)]
pub(crate) mod config {
pub(crate) use rustfs_ecstore::config::com;
}
pub(crate) mod disk {
pub(crate) use rustfs_ecstore::disk::{
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
UpdateMetadataOpts, WalkDirOptions, error,
};
}
pub(crate) mod error {
pub(crate) use rustfs_ecstore::error::{
Error, Result, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
}
pub(crate) mod global {
pub(crate) use rustfs_ecstore::global::{GLOBAL_TierConfigMgr, get_global_region};
}
pub(crate) mod metrics_realtime {
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
}
pub(crate) mod rio {
pub(crate) use rustfs_ecstore::rio::WriteEncryption;
}
pub(crate) mod rpc {
pub(crate) use rustfs_ecstore::rpc::{
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, verify_rpc_signature,
};
}
pub(crate) mod set_disk {
pub(crate) use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
}
pub(crate) mod store {
pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk_path, find_local_disk_by_ref};
}
pub(crate) type GetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
pub(crate) type ObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub(crate) type ObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub(crate) type PutObjReader = rustfs_ecstore::store_api::PutObjReader;
}
pub(crate) type GetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
pub(crate) type ObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub(crate) type ObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub(crate) type PutObjReader = rustfs_ecstore::store_api::PutObjReader;
+21 -59
View File
@@ -12,62 +12,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub(crate) mod ecstore {
pub(crate) use rustfs_ecstore::global::{resolve_object_store_handle, set_global_endpoints, update_erasure_type};
pub(crate) mod bucket {
pub(crate) use rustfs_ecstore::bucket::{metadata, metadata_sys, migration, quota, replication};
}
pub(crate) mod config {
pub(crate) use rustfs_ecstore::config::{com, init, init_global_config_sys, try_migrate_server_config};
}
pub(crate) mod disk {
pub(crate) use rustfs_ecstore::disk::{DiskAPI, RUSTFS_META_BUCKET, endpoint};
}
#[cfg(test)]
pub(crate) mod disks_layout {
pub(crate) use rustfs_ecstore::disks_layout::DisksLayout;
}
pub(crate) mod endpoints {
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
#[cfg(test)]
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
}
pub(crate) mod error {
pub(crate) use rustfs_ecstore::error::{Error, Result, StorageError};
}
pub(crate) mod event_notification {
pub(crate) use rustfs_ecstore::event_notification::{EventArgs, register_event_dispatch_hook};
}
pub(crate) mod global {
pub(crate) use rustfs_ecstore::global::{
get_global_endpoints_opt, get_global_lock_clients, get_global_region, is_dist_erasure, set_global_region,
set_global_rustfs_port, shutdown_background_services,
};
}
pub(crate) mod notification_sys {
pub(crate) use rustfs_ecstore::notification_sys::new_global_notification_sys;
}
pub(crate) mod rpc {
pub(crate) use rustfs_ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
}
pub(crate) mod set_disk {
pub(crate) use rustfs_ecstore::set_disk::get_lock_acquire_timeout;
}
pub(crate) mod store {
pub(crate) use rustfs_ecstore::store::{
ECStore, all_local_disk, init_local_disks, init_lock_clients, prewarm_local_disk_id_map,
};
}
}
pub(crate) use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
pub(crate) use rustfs_ecstore::bucket::migration::{try_migrate_bucket_metadata, try_migrate_iam_config};
pub(crate) use rustfs_ecstore::bucket::replication::{get_global_replication_pool, init_background_replication};
pub(crate) use rustfs_ecstore::bucket::{metadata, metadata_sys, quota};
pub(crate) use rustfs_ecstore::config::{com, init, init_global_config_sys, try_migrate_server_config};
pub(crate) use rustfs_ecstore::disk::{DiskAPI, RUSTFS_META_BUCKET, endpoint::Endpoint};
#[cfg(test)]
pub(crate) use rustfs_ecstore::disks_layout::DisksLayout;
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
#[cfg(test)]
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::error::{Error as EcstoreError, Result as EcstoreResult, StorageError};
pub(crate) use rustfs_ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook};
pub(crate) use rustfs_ecstore::global::{
get_global_endpoints_opt, get_global_lock_clients, get_global_region, is_dist_erasure, resolve_object_store_handle,
set_global_endpoints, set_global_region, set_global_rustfs_port, shutdown_background_services, update_erasure_type,
};
pub(crate) use rustfs_ecstore::notification_sys::new_global_notification_sys;
pub(crate) use rustfs_ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
pub(crate) use rustfs_ecstore::set_disk::get_lock_acquire_timeout;
pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk, init_local_disks, init_lock_clients, prewarm_local_disk_id_map};
+12 -12
View File
@@ -27,16 +27,16 @@ use std::{
time::{Duration as StdDuration, Instant},
};
use crate::storage_compat::ecstore::bucket::{
use crate::storage_compat::RUSTFS_META_BUCKET;
use crate::storage_compat::get_lock_acquire_timeout;
use crate::storage_compat::{EcstoreError, StorageError};
use crate::storage_compat::{
metadata::{
BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG,
BUCKET_TABLE_RESERVED_PREFIX, table_catalog_path_hash,
},
metadata_sys,
};
use crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET;
use crate::storage_compat::ecstore::error::{Error as EcstoreError, StorageError};
use crate::storage_compat::ecstore::set_disk::get_lock_acquire_timeout;
use bytes::Bytes;
use datafusion::{
arrow::datatypes::SchemaRef,
@@ -7813,7 +7813,7 @@ mod tests {
.map(|(bucket, _)| bucket.as_str())
.collect::<BTreeSet<_>>();
assert_eq!(object_buckets, BTreeSet::from([crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET]));
assert_eq!(object_buckets, BTreeSet::from([crate::storage_compat::RUSTFS_META_BUCKET]));
}
#[tokio::test]
@@ -10777,7 +10777,7 @@ mod tests {
.unwrap();
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &idempotency_path, 1)
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &idempotency_path, 1)
.await;
let err = store
@@ -10827,7 +10827,7 @@ mod tests {
.unwrap();
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
.await;
let request = TableCommitRequest {
@@ -10882,7 +10882,7 @@ mod tests {
.unwrap();
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
.await;
let result = store
@@ -10936,7 +10936,7 @@ mod tests {
.unwrap();
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
.await;
store
@@ -10990,7 +10990,7 @@ mod tests {
.unwrap();
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
.await;
store
@@ -11041,7 +11041,7 @@ mod tests {
backend.seed_object(bucket, &current_metadata, b"{}".to_vec()).await;
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &table_path, 2)
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &table_path, 2)
.await;
let err = store
@@ -11098,7 +11098,7 @@ mod tests {
.unwrap();
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &idempotency_path, 2)
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &idempotency_path, 2)
.await;
let result = store
@@ -65,6 +65,10 @@ STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_list_c
STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_operation_consumer_hits.txt"
STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE="${TMP_DIR}/store_api_object_operation_local_method_hits.txt"
DIRECT_ECSTORE_IMPORT_HITS_FILE="${TMP_DIR}/direct_ecstore_import_hits.txt"
TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/test_harness_nested_storage_compat_hits.txt"
RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/rustfs_nested_storage_compat_hits.txt"
RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/rustfs_runtime_scalar_storage_compat_hits.txt"
RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/rustfs_runtime_secondary_storage_compat_hits.txt"
PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE="${TMP_DIR}/production_unused_compat_allow_hits.txt"
BROAD_STORE_API_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/broad_store_api_compat_reexport_hits.txt"
NESTED_STORE_API_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/nested_store_api_compat_module_hits.txt"
@@ -204,10 +208,18 @@ require_source_line \
"crates/storage-api/src/lib.rs" \
"pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};" \
"storage-api public bucket contract re-export"
require_source_line \
"crates/storage-api/src/lib.rs" \
"pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};" \
"storage-api public capability contract re-export"
require_source_line \
"crates/storage-api/src/lib.rs" \
"pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};" \
"storage-api public multipart DTO re-export"
require_source_line \
"crates/storage-api/src/lib.rs" \
"pub use observability::{" \
"storage-api public observability contract re-export"
require_source_line \
"crates/storage-api/src/lib.rs" \
"pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};" \
@@ -240,6 +252,10 @@ require_source_line \
"crates/storage-api/src/lib.rs" \
"pub use error::{StorageErrorCode, StorageResult};" \
"storage-api public error contract re-export"
require_source_line \
"crates/storage-api/src/lib.rs" \
"pub use topology::{" \
"storage-api public topology contract re-export"
(
cd "$ROOT_DIR"
@@ -403,6 +419,66 @@ if [[ -s "$DIRECT_ECSTORE_IMPORT_HITS_FILE" ]]; then
report_failure "direct rustfs_ecstore imports outside compatibility boundaries are forbidden: $(paste -sd '; ' "$DIRECT_ECSTORE_IMPORT_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --no-heading '\bstorage_compat::ecstore\b|\bmod\s+ecstore\b' \
crates/e2e_test/src crates/heal/tests crates/scanner/tests fuzz/fuzz_targets \
--glob '*.rs' \
--glob '!target/**' || true
) >"$TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE"
if [[ -s "$TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE" ]]; then
report_failure "test and fuzz storage compatibility harnesses must use direct aliases instead of nested ecstore modules: $(paste -sd '; ' "$TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --no-heading '\bstorage_compat::ecstore\b|\bmod\s+ecstore\b' \
rustfs/src \
--glob '*.rs' \
--glob '!target/**' || true
) >"$RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE"
if [[ -s "$RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE" ]]; then
report_failure "RustFS runtime storage compatibility paths must use direct aliases instead of nested ecstore modules: $(paste -sd '; ' "$RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --no-heading '\bstorage_compat::(?:store|error|global|endpoints|set_disk|rpc|metrics_realtime|notification_sys|admin_server_info|store_utils|data_usage|disks_layout|event_notification)::|\bstorage_compat::disk::(?:RUSTFS_META_BUCKET|DiskAPI|endpoint::Endpoint)\b|\bstorage_compat::\{[^}\n]*(?:store|error|global|endpoints|set_disk|rpc|metrics_realtime|notification_sys|admin_server_info|store_utils|data_usage|disks_layout|event_notification|disk)::' \
rustfs/src \
--glob '*.rs' \
--glob '!target/**' || true
) >"$RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE"
if [[ -s "$RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE" ]]; then
report_failure "RustFS runtime scalar storage compatibility paths must use direct aliases instead of secondary modules: $(paste -sd '; ' "$RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
{
rg -n --no-heading '\bstorage_compat::(?:bucket|config|rio|client|tier|compress|disk|rebalance)::' \
rustfs/src \
--glob '*.rs' \
--glob '!target/**' || true
rg -n --no-heading '\bpub\(crate\)\s+mod\s+(?:bucket|config|rio|client|tier|compress|disk|rebalance)\b' \
rustfs/src \
--glob '*storage_compat.rs' \
--glob '!target/**' || true
find rustfs/src -type f -name '*.rs' -print0 |
xargs -0 perl -0ne '
while (/use\s+crate::(?:app::|admin::|storage::)?storage_compat::\{[^;]*?\b(bucket|config|rio|client|tier|compress|disk|rebalance)::/sg) {
print "$ARGV:grouped storage_compat::$1 import\n";
}
'
}
) >"$RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE"
if [[ -s "$RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE" ]]; then
report_failure "RustFS runtime secondary storage compatibility paths must use direct aliases instead of bucket/config/rio/client/tier/compress/disk/rebalance modules: $(paste -sd '; ' "$RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --no-heading '#!\[allow\(unused_imports\)\]' \