refactor: flatten test harness storage compat aliases (#3596)

* refactor: flatten test harness storage compat aliases

* refactor: flatten rustfs storage compat aliases (#3597)

* refactor: prune runtime storage compat surface (#3598)

* refactor: flatten runtime secondary storage compat (#3599)

* docs: add scheduler placement profiling baselines (#3600)

* feat: add observability topology capability contracts (#3601)
This commit is contained in:
安正超
2026-06-19 08:30:47 +08:00
committed by GitHub
parent b1c6578df1
commit ada6f7587e
87 changed files with 1551 additions and 990 deletions
@@ -15,7 +15,7 @@
// Used by test_distributed_lock_4_nodes_grpc in lock.rs // Used by test_distributed_lock_4_nodes_grpc in lock.rs
#![allow(dead_code)] #![allow(dead_code)]
use crate::storage_compat::ecstore::rpc::node_service_time_out_client_no_auth; use crate::storage_compat::node_service_time_out_client_no_auth;
use async_trait::async_trait; use async_trait::async_trait;
use rustfs_lock::{ use rustfs_lock::{
LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result, LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
@@ -41,10 +41,7 @@ impl GrpcLockClient {
&self, &self,
) -> Result< ) -> Result<
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient< rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
tonic::service::interceptor::InterceptedService< tonic::service::interceptor::InterceptedService<tonic::transport::Channel, crate::storage_compat::TonicInterceptor>,
tonic::transport::Channel,
crate::storage_compat::ecstore::rpc::TonicInterceptor,
>,
>, >,
> { > {
node_service_time_out_client_no_auth(&self.addr) node_service_time_out_client_no_auth(&self.addr)
@@ -14,8 +14,8 @@
// limitations under the License. // limitations under the License.
use crate::common::workspace_root; use crate::common::workspace_root;
use crate::storage_compat::ecstore::disk::{VolumeInfo, WalkDirOptions}; use crate::storage_compat::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
use crate::storage_compat::ecstore::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client}; use crate::storage_compat::{VolumeInfo, WalkDirOptions};
use futures::future::join_all; use futures::future::join_all;
use rmp_serde::{Deserializer, Serializer}; use rmp_serde::{Deserializer, Serializer};
use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter}; use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
@@ -15,7 +15,7 @@
use crate::common::{ use crate::common::{
RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client, RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
}; };
use crate::storage_compat::ecstore::bucket::bucket_target_sys::BucketTargetSys; use crate::storage_compat::BucketTargetSys;
use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
+8 -17
View File
@@ -12,22 +12,13 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
pub(crate) mod ecstore { #![allow(dead_code, unused_imports)]
#![allow(unused_imports)]
pub(crate) mod bucket { pub(crate) type BucketTargetSys = rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
pub(crate) mod bucket_target_sys { pub(crate) type TonicInterceptor = rustfs_ecstore::rpc::TonicInterceptor;
pub(crate) use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys; pub(crate) type VolumeInfo = rustfs_ecstore::disk::VolumeInfo;
} pub(crate) type WalkDirOptions = rustfs_ecstore::disk::WalkDirOptions;
}
pub(crate) mod disk { pub(crate) use rustfs_ecstore::rpc::{
pub(crate) use rustfs_ecstore::disk::{VolumeInfo, WalkDirOptions}; gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
} };
pub(crate) mod rpc {
pub(crate) use rustfs_ecstore::rpc::{
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
};
}
}
+6 -24
View File
@@ -12,28 +12,10 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
pub(crate) mod ecstore { #![allow(unused_imports)]
#![allow(unused_imports)]
pub(crate) mod bucket { pub(crate) use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
pub(crate) mod metadata_sys { pub(crate) use rustfs_ecstore::disk::{DiskStore, endpoint::Endpoint};
pub(crate) use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys; pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
} pub(crate) use rustfs_ecstore::store::ECStore;
} pub(crate) use rustfs_ecstore::store::init_local_disks;
pub(crate) mod disk {
pub(crate) use rustfs_ecstore::disk::DiskStore;
pub(crate) mod endpoint {
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
}
}
pub(crate) mod endpoints {
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
}
pub(crate) mod store {
pub(crate) use rustfs_ecstore::store::{ECStore, init_local_disks};
}
}
+3 -6
View File
@@ -16,8 +16,7 @@
mod common; mod common;
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint; use crate::common::storage_compat::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks};
use crate::common::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
use std::net::SocketAddr; use std::net::SocketAddr;
use tempfile::TempDir; use tempfile::TempDir;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
@@ -73,12 +72,10 @@ async fn test_endpoint_index_settings() -> anyhow::Result<()> {
} }
// test ECStore initialization // test ECStore initialization
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone()).await?; init_local_disks(endpoint_pools.clone()).await?;
let server_addr: SocketAddr = "127.0.0.1:0".parse().unwrap(); let server_addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
let ecstore = let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()).await?;
crate::common::storage_compat::ecstore::store::ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
.await?;
println!("ECStore initialized successfully with {} pools", ecstore.pools.len()); println!("ECStore initialized successfully with {} pools", ecstore.pools.len());
+8 -23
View File
@@ -22,7 +22,7 @@ use rustfs_heal::heal::{
#[test] #[test]
fn test_heal_event_to_heal_request_no_panic() { fn test_heal_event_to_heal_request_no_panic() {
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint; use crate::common::storage_compat::Endpoint;
// Test that invalid pool/set indices don't cause panic // Test that invalid pool/set indices don't cause panic
// Create endpoint using try_from or similar method // Create endpoint using try_from or similar method
@@ -47,7 +47,7 @@ fn test_heal_event_to_heal_request_no_panic() {
#[test] #[test]
fn test_heal_event_to_heal_request_valid_indices() { fn test_heal_event_to_heal_request_valid_indices() {
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint; use crate::common::storage_compat::Endpoint;
// Test that valid indices work correctly // Test that valid indices work correctly
let endpoint_result = Endpoint::try_from("http://localhost:9000"); let endpoint_result = Endpoint::try_from("http://localhost:9000");
@@ -192,14 +192,11 @@ fn test_heal_task_status_atomic_update() {
} }
async fn get_disk_status( async fn get_disk_status(
&self, &self,
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint, _endpoint: &crate::common::storage_compat::Endpoint,
) -> rustfs_heal::Result<rustfs_heal::heal::storage::DiskStatus> { ) -> rustfs_heal::Result<rustfs_heal::heal::storage::DiskStatus> {
Ok(rustfs_heal::heal::storage::DiskStatus::Ok) Ok(rustfs_heal::heal::storage::DiskStatus::Ok)
} }
async fn format_disk( async fn format_disk(&self, _endpoint: &crate::common::storage_compat::Endpoint) -> rustfs_heal::Result<()> {
&self,
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
) -> rustfs_heal::Result<()> {
Ok(()) Ok(())
} }
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_storage_api::BucketInfo>> { async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_storage_api::BucketInfo>> {
@@ -253,10 +250,7 @@ fn test_heal_task_status_atomic_update() {
) -> rustfs_heal::Result<(Vec<String>, Option<String>, bool)> { ) -> rustfs_heal::Result<(Vec<String>, Option<String>, bool)> {
Ok((vec![], None, false)) Ok((vec![], None, false))
} }
async fn get_disk_for_resume( async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<crate::common::storage_compat::DiskStore> {
&self,
_set_disk_id: &str,
) -> rustfs_heal::Result<crate::common::storage_compat::ecstore::disk::DiskStore> {
Err(rustfs_heal::Error::other("Not implemented in mock")) Err(rustfs_heal::Error::other("Not implemented in mock"))
} }
} }
@@ -326,17 +320,11 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
Ok(Vec::new()) Ok(Vec::new())
} }
async fn get_disk_status( async fn get_disk_status(&self, _endpoint: &crate::common::storage_compat::Endpoint) -> rustfs_heal::Result<DiskStatus> {
&self,
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
) -> rustfs_heal::Result<DiskStatus> {
Ok(DiskStatus::Ok) Ok(DiskStatus::Ok)
} }
async fn format_disk( async fn format_disk(&self, _endpoint: &crate::common::storage_compat::Endpoint) -> rustfs_heal::Result<()> {
&self,
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
) -> rustfs_heal::Result<()> {
Ok(()) Ok(())
} }
@@ -406,10 +394,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
Ok((Vec::new(), None, false)) Ok((Vec::new(), None, false))
} }
async fn get_disk_for_resume( async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<crate::common::storage_compat::DiskStore> {
&self,
_set_disk_id: &str,
) -> rustfs_heal::Result<crate::common::storage_compat::ecstore::disk::DiskStore> {
Err(rustfs_heal::Error::other("not implemented")) Err(rustfs_heal::Error::other("not implemented"))
} }
} }
+4 -8
View File
@@ -14,10 +14,8 @@
mod common; mod common;
use crate::common::storage_compat::ecstore::{ use crate::common::storage_compat::{
disk::endpoint::Endpoint, ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
store::ECStore,
}; };
use http::HeaderMap; use http::HeaderMap;
use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_common::heal_channel::{HealOpts, HealScanMode};
@@ -124,9 +122,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
// format disks (only first time) // format disks (only first time)
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone()) init_local_disks(endpoint_pools.clone()).await.unwrap();
.await
.unwrap();
// create ECStore with dynamic port 0 (let OS assign) or fixed 9001 if free // create ECStore with dynamic port 0 (let OS assign) or fixed 9001 if free
let port = 9001; // for simplicity let port = 9001; // for simplicity
@@ -144,7 +140,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
.await .await
.unwrap(); .unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect(); let buckets = buckets_list.into_iter().map(|v| v.name).collect();
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await; init_bucket_metadata_sys(ecstore.clone(), buckets).await;
// Create heal storage layer // Create heal storage layer
let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone())); let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
+21 -64
View File
@@ -12,70 +12,27 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
pub(crate) mod ecstore { #![allow(dead_code, unused_imports)]
#![allow(unused_imports)]
pub(crate) mod bucket { pub(crate) const BUCKET_LIFECYCLE_CONFIG: &str = rustfs_ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
pub(crate) mod lifecycle { pub(crate) const STORAGE_FORMAT_FILE: &str = rustfs_ecstore::disk::STORAGE_FORMAT_FILE;
pub(crate) use rustfs_ecstore::bucket::lifecycle::lifecycle::TransitionOptions;
pub(crate) mod bucket_lifecycle_ops { pub(crate) type TransitionOptions = rustfs_ecstore::bucket::lifecycle::lifecycle::TransitionOptions;
pub(crate) use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{
enqueue_transition_for_existing_objects, init_background_expiry,
};
}
}
pub(crate) mod metadata { pub(crate) use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{
pub(crate) use rustfs_ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG; enqueue_transition_for_existing_objects, init_background_expiry,
} };
pub(crate) use rustfs_ecstore::bucket::metadata_sys::{
pub(crate) mod metadata_sys { get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata,
pub(crate) use rustfs_ecstore::bucket::metadata_sys::{get, init_bucket_metadata_sys, update}; };
} pub(crate) use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
pub(crate) use rustfs_ecstore::client::transition_api::{ReadCloser, ReaderImpl};
pub(crate) mod versioning_sys { pub(crate) use rustfs_ecstore::disk::{DiskAPI, DiskOption, endpoint::Endpoint, new_disk};
pub(crate) use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys; pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
} pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
} pub(crate) use rustfs_ecstore::pools::path2_bucket_object_with_base_path;
pub(crate) use rustfs_ecstore::store::ECStore;
pub(crate) mod client { pub(crate) use rustfs_ecstore::store::init_local_disks;
pub(crate) mod transition_api { pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierMinIO, TierType};
pub(crate) use rustfs_ecstore::client::transition_api::{ReadCloser, ReaderImpl}; pub(crate) use rustfs_ecstore::tier::warm_backend::WarmBackendGetOpts;
} pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, build_transition_put_options};
}
pub(crate) mod disk {
pub(crate) use rustfs_ecstore::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, new_disk};
pub(crate) mod endpoint {
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
}
}
pub(crate) mod endpoints {
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
}
pub(crate) mod global {
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
}
pub(crate) mod pools {
pub(crate) use rustfs_ecstore::pools::path2_bucket_object_with_base_path;
}
pub(crate) mod store {
pub(crate) use rustfs_ecstore::store::{ECStore, init_local_disks};
}
pub(crate) mod tier {
pub(crate) mod tier_config {
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierMinIO, TierType};
}
pub(crate) mod warm_backend {
pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options};
}
}
}
@@ -14,24 +14,12 @@
mod common; mod common;
use crate::common::storage_compat::ecstore::{ use crate::common::storage_compat::{
bucket::metadata::BUCKET_LIFECYCLE_CONFIG, BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DiskAPI, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
bucket::{ GLOBAL_TierConfigMgr, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, TierConfig, TierMinIO, TierType,
lifecycle::{TransitionOptions, bucket_lifecycle_ops::enqueue_transition_for_existing_objects}, TransitionOptions, WarmBackend, WarmBackendGetOpts, build_transition_put_options, enqueue_transition_for_existing_objects,
metadata_sys, get_bucket_metadata, init_background_expiry, init_bucket_metadata_sys, init_local_disks, new_disk,
versioning_sys::BucketVersioningSys, path2_bucket_object_with_base_path, update_bucket_metadata,
},
client::transition_api::{ReadCloser, ReaderImpl},
disk::endpoint::Endpoint,
disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, new_disk},
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
global::GLOBAL_TierConfigMgr,
pools::path2_bucket_object_with_base_path,
store::ECStore,
tier::{
tier_config::{TierConfig, TierMinIO, TierType},
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
},
}; };
use futures::FutureExt; use futures::FutureExt;
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT; use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
@@ -125,9 +113,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
// format disks (only first time) // format disks (only first time)
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone()) init_local_disks(endpoint_pools.clone()).await.unwrap();
.await
.unwrap();
// create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free // create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
let port = 9002; // for simplicity let port = 9002; // for simplicity
@@ -145,11 +131,10 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
.await .await
.unwrap(); .unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect(); let buckets = buckets_list.into_iter().map(|v| v.name).collect();
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await; init_bucket_metadata_sys(ecstore.clone(), buckets).await;
// Initialize background expiry workers // Initialize background expiry workers
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()) init_background_expiry(ecstore.clone()).await;
.await;
// Store in global once lock // Store in global once lock
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone())); let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
@@ -197,9 +182,7 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
}; };
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone()) init_local_disks(endpoint_pools.clone()).await.unwrap();
.await
.unwrap();
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap(); let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()) let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
@@ -214,11 +197,10 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
.await .await
.unwrap(); .unwrap();
let buckets = buckets_list.into_iter().map(|v| v.name).collect(); let buckets = buckets_list.into_iter().map(|v| v.name).collect();
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await; init_bucket_metadata_sys(ecstore.clone(), buckets).await;
if init_expiry { if init_expiry {
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()) init_background_expiry(ecstore.clone()).await;
.await;
} }
(disk_paths, ecstore) (disk_paths, ecstore)
@@ -287,7 +269,7 @@ async fn set_bucket_lifecycle(bucket_name: &str) -> Result<(), Box<dyn std::erro
</Rule> </Rule>
</LifecycleConfiguration>"#; </LifecycleConfiguration>"#;
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?; update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
Ok(()) Ok(())
} }
@@ -311,7 +293,7 @@ async fn set_bucket_lifecycle_deletemarker(bucket_name: &str) -> Result<(), Box<
</Rule> </Rule>
</LifecycleConfiguration>"#; </LifecycleConfiguration>"#;
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?; update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
Ok(()) Ok(())
} }
@@ -334,7 +316,7 @@ async fn set_bucket_lifecycle_delmarker_expiration(bucket_name: &str, days: i64)
</LifecycleConfiguration>"# </LifecycleConfiguration>"#
); );
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?; update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
Ok(()) Ok(())
} }
@@ -378,7 +360,7 @@ async fn set_bucket_lifecycle_transition_with_tier(
</LifecycleConfiguration>"# </LifecycleConfiguration>"#
); );
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?; update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
Ok(()) Ok(())
} }
@@ -613,7 +595,7 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
.await .await
.expect("failed to stat object metadata") .expect("failed to stat object metadata")
.file_type(); .file_type();
let lifecycle = metadata_sys::get(bucket) let lifecycle = get_bucket_metadata(bucket)
.await .await
.expect("failed to load bucket metadata") .expect("failed to load bucket metadata")
.lifecycle_config .lifecycle_config
@@ -878,7 +860,7 @@ mod serial_tests {
println!("✅ Lifecycle configuration set for bucket: {bucket_name}"); println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
// Verify lifecycle configuration was set // Verify lifecycle configuration was set
match crate::common::storage_compat::ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await { match get_bucket_metadata(bucket_name.as_str()).await {
Ok(bucket_meta) => { Ok(bucket_meta) => {
assert!(bucket_meta.lifecycle_config.is_some()); assert!(bucket_meta.lifecycle_config.is_some());
println!("✅ Bucket metadata retrieved successfully"); println!("✅ Bucket metadata retrieved successfully");
@@ -1284,8 +1266,7 @@ mod serial_tests {
"stale transitioned remote object should still exist before scanner fallback runs" "stale transitioned remote object should still exist before scanner fallback runs"
); );
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()) init_background_expiry(ecstore.clone()).await;
.await;
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await; scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!( assert!(
@@ -1346,8 +1327,7 @@ mod serial_tests {
"stale transitioned remote object should still exist before scanner cleanup runs" "stale transitioned remote object should still exist before scanner cleanup runs"
); );
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()) init_background_expiry(ecstore.clone()).await;
.await;
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await; scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!( assert!(
@@ -1437,7 +1417,7 @@ mod serial_tests {
</Rule> </Rule>
</LifecycleConfiguration>"# </LifecycleConfiguration>"#
); );
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await .await
.expect("Failed to set lifecycle configuration"); .expect("Failed to set lifecycle configuration");
@@ -1522,7 +1502,7 @@ mod serial_tests {
</Rule> </Rule>
</LifecycleConfiguration>"# </LifecycleConfiguration>"#
); );
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await .await
.expect("Failed to set lifecycle configuration"); .expect("Failed to set lifecycle configuration");
@@ -1714,8 +1694,7 @@ mod serial_tests {
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await); assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()) init_background_expiry(ecstore.clone()).await;
.await;
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await; scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
assert!( assert!(
@@ -1750,7 +1729,7 @@ mod serial_tests {
</Rule> </Rule>
</LifecycleConfiguration>"# </LifecycleConfiguration>"#
); );
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await .await
.expect("Failed to set lifecycle configuration"); .expect("Failed to set lifecycle configuration");
@@ -1815,12 +1794,11 @@ mod serial_tests {
</NoncurrentVersionExpiration> </NoncurrentVersionExpiration>
</Rule> </Rule>
</LifecycleConfiguration>"#; </LifecycleConfiguration>"#;
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()) update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
.await .await
.expect("Failed to set noncurrent lifecycle configuration"); .expect("Failed to set noncurrent lifecycle configuration");
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()) init_background_expiry(ecstore.clone()).await;
.await;
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await; scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
@@ -1854,7 +1832,7 @@ mod serial_tests {
</NoncurrentVersionExpiration> </NoncurrentVersionExpiration>
</Rule> </Rule>
</LifecycleConfiguration>"#; </LifecycleConfiguration>"#;
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()) update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
.await .await
.expect("Failed to set noncurrent lifecycle configuration"); .expect("Failed to set noncurrent lifecycle configuration");
@@ -1941,7 +1919,7 @@ mod serial_tests {
</Rule> </Rule>
</LifecycleConfiguration>"# </LifecycleConfiguration>"#
); );
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()) update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
.await .await
.expect("Failed to set lifecycle configuration"); .expect("Failed to set lifecycle configuration");
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await; upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
+113
View File
@@ -0,0 +1,113 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt;
use serde::{Deserialize, Serialize};
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CapabilityState {
Supported,
Unsupported,
Disabled,
#[default]
Unknown,
}
impl CapabilityState {
pub const fn is_supported(self) -> bool {
matches!(self, Self::Supported)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CapabilityStatus {
pub state: CapabilityState,
#[serde(skip_serializing_if = "Option::is_none")]
pub reason: Option<String>,
}
impl CapabilityStatus {
pub const fn new(state: CapabilityState) -> Self {
Self { state, reason: None }
}
pub const fn supported() -> Self {
Self::new(CapabilityState::Supported)
}
pub const fn unsupported() -> Self {
Self::new(CapabilityState::Unsupported)
}
pub const fn disabled() -> Self {
Self::new(CapabilityState::Disabled)
}
pub const fn unknown() -> Self {
Self::new(CapabilityState::Unknown)
}
pub fn with_reason(mut self, reason: impl Into<String>) -> Self {
self.reason = Some(reason.into());
self
}
}
impl Default for CapabilityStatus {
fn default() -> Self {
Self::unknown()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CapabilitySnapshotError {
Unavailable,
Unsupported,
InvalidSnapshot(String),
}
impl fmt::Display for CapabilitySnapshotError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Unavailable => f.write_str("capability snapshot unavailable"),
Self::Unsupported => f.write_str("capability snapshot unsupported"),
Self::InvalidSnapshot(reason) => write!(f, "invalid capability snapshot: {reason}"),
}
}
}
impl std::error::Error for CapabilitySnapshotError {}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn capability_status_serializes_unknown_and_unsupported_states() {
let unknown = CapabilityStatus::unknown();
let unsupported = CapabilityStatus::unsupported().with_reason("target does not expose profiler");
let encoded = serde_json::to_string(&(unknown, unsupported)).expect("serialize capability statuses");
let decoded: (CapabilityStatus, CapabilityStatus) =
serde_json::from_str(&encoded).expect("deserialize capability statuses");
assert_eq!(decoded.0.state, CapabilityState::Unknown);
assert_eq!(decoded.1.state, CapabilityState::Unsupported);
assert_eq!(decoded.1.reason.as_deref(), Some("target does not expose profiler"));
assert!(!decoded.1.state.is_supported());
}
}
+11
View File
@@ -16,12 +16,16 @@
pub mod admin; pub mod admin;
pub mod bucket; pub mod bucket;
pub mod capability;
pub mod error; pub mod error;
pub mod multipart; pub mod multipart;
pub mod object; pub mod object;
pub mod observability;
pub mod topology;
pub use admin::{DiskSetSelector, StorageAdminApi}; pub use admin::{DiskSetSelector, StorageAdminApi};
pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp}; pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
pub use error::{StorageErrorCode, StorageResult}; pub use error::{StorageErrorCode, StorageResult};
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo}; pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
pub use object::{DeletedObject, ObjectToDelete}; pub use object::{DeletedObject, ObjectToDelete};
@@ -31,3 +35,10 @@ pub use object::{HealOperations, MultipartOperations, NamespaceLocking, ObjectIO
pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr}; pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};
pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState}; pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};
pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder}; pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};
pub use observability::{
MemorySamplingState, ObservabilitySnapshot, ObservabilitySnapshotProvider, PlatformSupport, UserspaceProfilingCapability,
};
pub use topology::{
DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet, TopologySnapshot,
TopologySnapshotProvider,
};
+102
View File
@@ -0,0 +1,102 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::fmt::Debug;
use serde::{Deserialize, Serialize};
use crate::{CapabilitySnapshotError, CapabilityStatus};
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ObservabilitySnapshot {
pub runtime_telemetry: CapabilityStatus,
pub userspace_profiling: UserspaceProfilingCapability,
pub memory_sampling: MemorySamplingState,
pub platform: PlatformSupport,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct UserspaceProfilingCapability {
pub cpu: CapabilityStatus,
pub memory: CapabilityStatus,
pub continuous_cpu: CapabilityStatus,
pub periodic_cpu: CapabilityStatus,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MemorySamplingState {
pub process: CapabilityStatus,
pub system: CapabilityStatus,
pub cgroup: CapabilityStatus,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct PlatformSupport {
pub target_triple: Option<String>,
pub os: Option<String>,
pub arch: Option<String>,
pub allocator: CapabilityStatus,
pub ebpf: CapabilityStatus,
pub numa: CapabilityStatus,
}
#[async_trait::async_trait]
pub trait ObservabilitySnapshotProvider: Send + Sync + Debug {
async fn observability_snapshot(&self) -> Result<ObservabilitySnapshot, CapabilitySnapshotError>;
}
#[cfg(test)]
mod tests {
use super::*;
use crate::CapabilityState;
#[test]
fn observability_snapshot_preserves_unknown_and_unsupported_states() {
let snapshot = ObservabilitySnapshot {
runtime_telemetry: CapabilityStatus::unknown(),
userspace_profiling: UserspaceProfilingCapability {
cpu: CapabilityStatus::unsupported().with_reason("unsupported target"),
memory: CapabilityStatus::disabled(),
continuous_cpu: CapabilityStatus::unknown(),
periodic_cpu: CapabilityStatus::supported(),
},
memory_sampling: MemorySamplingState {
process: CapabilityStatus::supported(),
system: CapabilityStatus::supported(),
cgroup: CapabilityStatus::unknown(),
},
platform: PlatformSupport {
target_triple: Some("x86_64-unknown-linux-gnu".to_owned()),
os: Some("linux".to_owned()),
arch: Some("x86_64".to_owned()),
allocator: CapabilityStatus::supported(),
ebpf: CapabilityStatus::unknown(),
numa: CapabilityStatus::unsupported(),
},
};
let encoded = serde_json::to_string(&snapshot).expect("serialize observability snapshot");
let decoded: ObservabilitySnapshot = serde_json::from_str(&encoded).expect("deserialize observability snapshot");
assert_eq!(decoded.runtime_telemetry.state, CapabilityState::Unknown);
assert_eq!(decoded.userspace_profiling.cpu.state, CapabilityState::Unsupported);
assert_eq!(decoded.userspace_profiling.memory.state, CapabilityState::Disabled);
assert_eq!(decoded.platform.numa.state, CapabilityState::Unsupported);
assert_eq!(decoded.platform.ebpf.state, CapabilityState::Unknown);
}
}
+153
View File
@@ -0,0 +1,153 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::BTreeMap;
use std::fmt::Debug;
use serde::{Deserialize, Serialize};
use crate::{CapabilitySnapshotError, CapabilityStatus};
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TopologySnapshot {
pub pools: Vec<TopologyPool>,
pub capabilities: TopologyCapabilities,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TopologyCapabilities {
pub profiling: CapabilityStatus,
pub numa: CapabilityStatus,
pub failure_domain_labels: CapabilityStatus,
pub media_labels: CapabilityStatus,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TopologyPool {
pub pool_index: usize,
pub pool_id: Option<String>,
pub labels: TopologyLabels,
pub sets: Vec<TopologySet>,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TopologySet {
pub pool_index: usize,
pub set_index: usize,
pub set_id: Option<String>,
pub labels: TopologyLabels,
pub disks: Vec<TopologyDisk>,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TopologyDisk {
pub pool_index: usize,
pub set_index: usize,
pub disk_index: usize,
pub disk_id: Option<String>,
pub labels: TopologyLabels,
pub capabilities: DiskCapabilities,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TopologyLabels {
pub zone: Option<String>,
pub rack: Option<String>,
pub node: Option<String>,
pub media: Option<String>,
pub numa_node: Option<String>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub additional: BTreeMap<String, String>,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DiskCapabilities {
pub media_type: CapabilityStatus,
pub failure_domain: CapabilityStatus,
pub numa: CapabilityStatus,
pub profiling: CapabilityStatus,
}
#[async_trait::async_trait]
pub trait TopologySnapshotProvider: Send + Sync + Debug {
async fn topology_snapshot(&self) -> Result<TopologySnapshot, CapabilitySnapshotError>;
}
#[cfg(test)]
mod tests {
use super::*;
use crate::CapabilityState;
#[test]
fn topology_snapshot_allows_missing_and_extra_labels() {
let raw = r#"{
"pools": [{
"pool_index": 0,
"pool_id": null,
"labels": {
"additional": {
"room": "a"
}
},
"sets": [{
"pool_index": 0,
"set_index": 1,
"set_id": null,
"labels": {},
"disks": [{
"pool_index": 0,
"set_index": 1,
"disk_index": 2,
"disk_id": "disk-2",
"labels": {
"media": "ssd",
"additional": {
"slot": "nvme0"
}
},
"capabilities": {
"media_type": { "state": "supported" },
"failure_domain": { "state": "unknown" },
"numa": { "state": "unsupported", "reason": "not reported" },
"profiling": { "state": "disabled" }
}
}]
}]
}],
"capabilities": {
"profiling": { "state": "supported" },
"numa": { "state": "unknown" },
"failure_domain_labels": { "state": "supported" },
"media_labels": { "state": "supported" }
}
}"#;
let snapshot: TopologySnapshot = serde_json::from_str(raw).expect("deserialize topology snapshot");
let disk = &snapshot.pools[0].sets[0].disks[0];
assert_eq!(snapshot.pools[0].labels.zone, None);
assert_eq!(snapshot.pools[0].labels.additional.get("room").map(String::as_str), Some("a"));
assert_eq!(disk.labels.media.as_deref(), Some("ssd"));
assert_eq!(disk.labels.additional.get("slot").map(String::as_str), Some("nvme0"));
assert_eq!(disk.capabilities.numa.state, CapabilityState::Unsupported);
assert_eq!(disk.capabilities.profiling.state, CapabilityState::Disabled);
}
}
+3
View File
@@ -92,14 +92,17 @@ Required `rustfs-storage-api` public re-exports:
- `pub use admin::{DiskSetSelector, StorageAdminApi};` - `pub use admin::{DiskSetSelector, StorageAdminApi};`
- `pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};` - `pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};`
- `pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};`
- `pub use error::{StorageErrorCode, StorageResult};` - `pub use error::{StorageErrorCode, StorageResult};`
- `pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};` - `pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};`
- `pub use observability::{MemorySamplingState, ObservabilitySnapshot, ObservabilitySnapshotProvider, PlatformSupport, UserspaceProfilingCapability};`
- `pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};` - `pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};`
- `pub use object::{ExpirationOptions, TransitionedObject};` - `pub use object::{ExpirationOptions, TransitionedObject};`
- `pub use object::{HealOperations, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations};` - `pub use object::{HealOperations, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations};`
- `pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};` - `pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};`
- `pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};` - `pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};`
- `pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};` - `pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};`
- `pub use topology::{DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet, TopologySnapshot, TopologySnapshotProvider};`
ECStore must keep compile-time coverage for `StorageAdminApi`, `HealOperations`, ECStore must keep compile-time coverage for `StorageAdminApi`, `HealOperations`,
and the separate `NamespaceLocking` operation group. and the separate `NamespaceLocking` operation group.
+224 -15
View File
@@ -5,17 +5,19 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context ## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-storage-api-lifecycle-contracts` - Branch: `overtrue/arch-observability-topology-contracts`
- Baseline: stacked on `rustfs/rustfs#3594` head - Baseline: stacked on `origin/overtrue/arch-test-harness-compat-aliases`
(`6fc05b84e2cd22a0482adfbc042184b03f8fdfa6`). after `rustfs/rustfs#3600`
- PR type for this branch: `pure-move` (`ae6a5befcf60f2f2ebce9799ba93649032234273`).
- Runtime behavior changes: no migration behavior change expected. - PR type for this branch: `contract`
- Rust code changes: move lifecycle helper DTO contracts for expiration and - Runtime behavior changes: none.
transitioned object metadata into rustfs-storage-api, switch ECStore internal - Rust code changes: add observability and topology capability DTO/trait
consumers to direct storage-api imports, and keep ECStore old-path re-exports. contracts to `rustfs-storage-api`.
- CI/script changes: add migration guards rejecting reintroduced ECStore - CI/script changes: extend migration re-export guard coverage for the new
lifecycle helper DTO definitions and old internal consumer imports. contract exports.
- Docs changes: record the lifecycle helper pure-move slice. - Docs changes: add
[`runtime-capability-contracts.md`](runtime-capability-contracts.md) and
record the combined PR-08/API-013 plus PR-09/API-014 contract slice.
## Phase 0 Tasks ## Phase 0 Tasks
@@ -63,6 +65,60 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
[`ecstore-config-consumer-inventory.md`](ecstore-config-consumer-inventory.md) [`ecstore-config-consumer-inventory.md`](ecstore-config-consumer-inventory.md)
records the current model definitions, global accessors, persistence helpers, records the current model definitions, global accessors, persistence helpers,
consumer groups, migration risks, and do-not-change contract. consumer groups, migration risks, and do-not-change contract.
- [x] `G-011` Inventory scheduler baseline.
- Acceptance:
[`scheduler-baseline.md`](scheduler-baseline.md) records current owners for
request admission, reusable scheduler/backpressure facades, workers, scanner
budget, heal admission, and the Tokio runtime builder.
- Must preserve: no Rust source changes, no scheduler/controller contract
changes, and no runtime behavior changes.
- [x] `G-012` Inventory placement and repair invariants.
- Acceptance:
[`placement-repair-invariants.md`](placement-repair-invariants.md) records
object-to-set hashing, pool/set/disk assignment boundaries, set-aware
readiness and lock quorum, scanner budget, and heal admission preservation
gates.
- Must preserve: no placement, repair, scanner, heal, readiness, lock, or
storage metadata behavior changes.
- [x] `G-013` Inventory profiling and NUMA capabilities.
- Acceptance:
[`profiling-numa-capability-inventory.md`](profiling-numa-capability-inventory.md)
records current CPU/memory profiling, cgroup memory sampling, allocator
backend, eBPF, and NUMA capability support plus no-op fallback invariants.
- Must preserve: no startup, profiling, allocator, runtime, or platform-gate
behavior changes.
## Issue #660 Capability Contract Tasks
- [x] `PR-08/API-013` Add observability snapshot contract.
- Completed slice: add `CapabilityState`, `CapabilityStatus`,
`CapabilitySnapshotError`, `ObservabilitySnapshot`,
`UserspaceProfilingCapability`, `MemorySamplingState`,
`PlatformSupport`, and `ObservabilitySnapshotProvider` to
`rustfs-storage-api`.
- Acceptance: runtime telemetry, userspace profiling, memory sampling, and
platform support states are representable without runtime, ECStore, admin,
profiling, exporter, sidecar, eBPF, or OTEL implementation dependencies.
- Must preserve: no profiling, startup, admin route, exporter, sidecar, eBPF,
OTEL, or runtime behavior changes.
- Verification: storage-api contract tests for unknown, unsupported,
disabled, and supported capability states; focused storage-api check;
migration guard; formatting; diff hygiene; and three-expert review.
- [x] `PR-09/API-014` Add topology capability contract.
- Completed slice: add `TopologySnapshot`, `TopologyCapabilities`,
`TopologyPool`, `TopologySet`, `TopologyDisk`, `TopologyLabels`,
`DiskCapabilities`, and `TopologySnapshotProvider` to
`rustfs-storage-api`.
- Acceptance: pool, set, and disk identity fields plus optional zone, rack,
node, media, NUMA, and additional labels are representable without
`rustfs-ecstore`.
- Must preserve: no ECStore endpoint/set implementation, placement,
membership, NUMA pinning, or runtime behavior changes.
- Verification: storage-api contract tests for missing and additional labels
plus supported, unsupported, unknown, and disabled capability states;
focused storage-api check; migration guard; formatting; diff hygiene; and
three-expert review.
- [x] `TEST-PRTYPE-001` Check PR type enum consistency. - [x] `TEST-PRTYPE-001` Check PR type enum consistency.
- Acceptance: `./scripts/check_architecture_migration_rules.sh` parses the - Acceptance: `./scripts/check_architecture_migration_rules.sh` parses the
allowed PR types from [`crate-boundaries.md`](crate-boundaries.md) and fails allowed PR types from [`crate-boundaries.md`](crate-boundaries.md) and fails
@@ -1296,6 +1352,87 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
migration and layer guards, formatting check, diff hygiene, risk scan, full migration and layer guards, formatting check, diff hygiene, risk scan, full
pre-commit, and required three-expert review passed before push. pre-commit, and required three-expert review passed before push.
- [x] `API-051` Flatten test harness storage compatibility aliases.
- Current branch: `overtrue/arch-test-harness-compat-aliases`.
- Current slice: flatten e2e, heal, scanner, and fuzz storage compatibility
harnesses from nested `storage_compat::ecstore` modules into direct
crate-local aliases, constants, and function imports.
- Acceptance: no e2e, heal-test, scanner-test, or fuzz-target harness file
may expose or consume nested `storage_compat::ecstore` paths, and migration
rules reject reintroducing nested test/fuzz ECStore compatibility modules.
- Must preserve: e2e bucket target/RPC/disk helper imports, heal ECStore disk
and endpoint setup, scanner lifecycle/tier/disk/storage setup, fuzz bucket
validation behavior, and fuzz path-containment validation behavior.
- Risk defense: this is test-harness and fuzz-harness import cleanup only; no
production runtime behavior, ECStore ownership, storage metadata format, or
scanner/heal lifecycle logic is changed.
- Verification: focused e2e/heal/scanner test compile, harness tests,
migration and layer guards, formatting check, diff hygiene, risk scan, full
pre-commit, and required three-expert review passed before push.
- [x] `API-052` Flatten RustFS runtime storage compatibility aliases.
- Current branch: `overtrue/arch-rustfs-storage-compat-aliases`.
- Current slice: flatten RustFS root, app, admin, and storage runtime
compatibility facades from nested `storage_compat::ecstore` modules into
direct crate-local aliases, constants, and function imports.
- Acceptance: no RustFS runtime source file may expose or consume nested
`storage_compat::ecstore` paths, and migration rules reject reintroducing
nested RustFS runtime ECStore compatibility modules.
- Must preserve: startup/config/bootstrap behavior, server readiness checks,
admin replication/rebalance/tier/config handlers, app object/bucket/
multipart usecases, storage RPC/SSE/access paths, table catalog storage
access, and existing local compatibility ownership.
- Risk defense: this is RustFS runtime import cleanup only; no production
runtime behavior, ECStore ownership, storage metadata format, object I/O,
admin authorization, or readiness semantics are changed.
- Verification: focused RustFS compile, migration and layer guards,
formatting check, diff hygiene, risk scan, full pre-commit, and required
three-expert review passed before push.
- [x] `API-053` Flatten RustFS runtime scalar storage compatibility aliases.
- Current branch: `overtrue/arch-runtime-compat-surface-prune`.
- Current slice: flatten RustFS root, app, admin, and storage runtime scalar
compatibility facades such as store, error, global, endpoints, RPC,
metrics, notification, set-disk, and data-usage paths into direct
crate-local aliases and functions.
- Acceptance: RustFS runtime source no longer consumes those scalar
compatibility surfaces through secondary modules, while higher-coupling
bucket/config/rio compatibility modules remain unchanged; migration rules
reject restoring the flattened scalar paths.
- Must preserve: startup config/bootstrap behavior, server readiness checks,
admin replication/rebalance/tier/config handlers, app object/bucket/
multipart usecases, storage RPC/SSE/access paths, table catalog storage
access, and existing ECStore concrete type ownership.
- Risk defense: this is import ownership and facade-shape cleanup only; no
production runtime behavior, ECStore ownership, storage metadata format,
object I/O, admin authorization, or readiness semantics are changed.
- Verification: focused RustFS compile, migration and layer guards,
formatting check, diff hygiene, risk scan, full pre-commit, and required
three-expert review passed before push.
- [x] `API-054` Flatten RustFS runtime secondary storage compatibility aliases.
- Current branch: `overtrue/arch-runtime-secondary-compat-flatten`.
- Current slice: flatten RustFS root, app, admin, and storage runtime
secondary compatibility modules such as bucket, config, rio, client, tier,
compress, disk, and rebalance into direct crate-local aliases, modules, and
functions.
- Acceptance: RustFS runtime source no longer consumes those compatibility
surfaces through broad secondary modules, the runtime compatibility files no
longer define those wrapper modules, and migration rules reject restoring
the flattened secondary paths.
- Must preserve: startup config/bootstrap behavior, server module-switch
config reads, embedded startup storage initialization, admin bucket/meta/
tier/rebalance/config handlers, app object/bucket/multipart usecases,
storage RPC/SSE/access paths, table catalog storage access, and ECStore
concrete type ownership.
- Risk defense: this is import ownership and facade-shape cleanup only; no
production runtime behavior, ECStore ownership, storage metadata format,
object I/O, admin authorization, tier behavior, or readiness semantics are
changed.
- Verification: focused RustFS compile, migration and layer guards,
formatting check, diff hygiene, risk scan, and required three-expert review
passed before push.
## Phase 8 Background Controller Tasks ## Phase 8 Background Controller Tasks
- [x] `BGC-001` Inventory background services. - [x] `BGC-001` Inventory background services.
@@ -1564,7 +1701,10 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Next PRs ## Next PRs
1. `pure-move`/`consumer-migration`: continue larger cleanup slices with the 1. `contract`/`consumer-migration`: wire read-only observability and topology
snapshots to implementation owners without changing runtime, profiling,
placement, or admin route behavior.
2. `pure-move`/`consumer-migration`: continue larger cleanup slices with the
loss-prevention guards active for remaining ECStore compatibility contracts loss-prevention guards active for remaining ECStore compatibility contracts
now that broad compatibility passthroughs are fully closed. now that broad compatibility passthroughs are fully closed.
@@ -1572,14 +1712,65 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
| Expert | Status | Notes | | Expert | Status | Notes |
|---|---|---| |---|---|---|
| Quality/architecture | passed | S-015 removes obsolete KMS admin policy action variants after the handler fallback cleanup; API-042/API-043/API-044/API-045/API-046/API-047/API-048/API-049/API-050 narrow notify, S3 Select, OBS, IAM, Swift, heal, scanner, RustFS runtime, test, fuzz, and lifecycle helper compatibility contracts without moving ECStore storage metadata ownership. | | Quality/architecture | passed | S-015 removes obsolete KMS admin policy action variants after the handler fallback cleanup; API-042/API-043/API-044/API-045/API-046/API-047/API-048/API-049/API-050/API-051/API-052/API-053/API-054 narrow notify, S3 Select, OBS, IAM, Swift, heal, scanner, RustFS runtime, test, fuzz, lifecycle helper, harness, and RustFS runtime compatibility contracts without moving ECStore storage metadata ownership; G-011/G-012/G-013 add docs-only baselines for scheduler, placement/repair, and profiling/NUMA work; Issue #660 PR-08/PR-09 add read-only observability and topology contracts in rustfs-storage-api only. |
| Migration preservation | passed | KMS endpoint URLs, query aliases, request bodies, response contracts, and dedicated `kms:*` authorization behavior are preserved; event builder call sites, ECStore event bridge conversion, restore event data, version IDs, metadata filtering, config read/save semantics, S3 Select store/error/buffer semantics, OBS metrics state reads, IAM config/notification/error semantics, Swift bucket metadata access, heal disk/resume/task behavior, scanner lifecycle/replication/data-usage behavior, RustFS startup/admin/app/storage runtime access, e2e/test/fuzz import behavior, lifecycle expiration/transition helper DTO field contracts, unchanged no-op handling, and remove-event behavior are preserved. | | Migration preservation | passed | KMS endpoint URLs, query aliases, request bodies, response contracts, and dedicated `kms:*` authorization behavior are preserved; event builder call sites, ECStore event bridge conversion, restore event data, version IDs, metadata filtering, config read/save semantics, S3 Select store/error/buffer semantics, OBS metrics state reads, IAM config/notification/error semantics, Swift bucket metadata access, heal disk/resume/task behavior, scanner lifecycle/replication/data-usage behavior, RustFS startup/admin/app/storage runtime access, e2e/test/fuzz import behavior, lifecycle expiration/transition helper DTO field contracts, flattened harness and RustFS runtime scalar/secondary alias behavior, unchanged no-op handling, remove-event behavior, scheduler/readiness/placement/profiling runtime behavior, platform gates, missing/unknown capability states, and placement/topology labels are preserved. |
| Testing/verification | passed | Focused compiles/tests, guards, formatting, diff hygiene, risk scan, and full `make pre-commit` passed for the current slice. | | Testing/verification | passed | Focused compiles/tests, fuzz target compile, guards, formatting, diff hygiene, risk scan, and full `make pre-commit` passed for prior code slices; current Issue #660 PR-08/PR-09 contract slice uses storage-api tests/checks, migration guard, formatting, diff hygiene, and three-expert review. |
## Verification Notes ## Verification Notes
Passed before push: Passed before push:
- Issue #660 PR-08/PR-09 current slice:
- `cargo test -p rustfs-storage-api`: passed.
- `cargo check -p rustfs-storage-api`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- `make pre-commit`: passed.
- Three-expert review: passed.
- G-011/G-012/G-013 current slice:
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `git diff --check`: passed.
- Three-expert review: passed.
- Full `make pre-commit`: not run because this slice is documentation-only.
- API-054 current slice:
- `cargo check -p rustfs --lib`: passed.
- `cargo check --tests -p rustfs`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- Rust risk scan: passed; only existing import and path rewrites were
reviewed, with no new unwrap/expect, panic/todo/unsafe, risky casts,
ad-hoc error construction, or sensitive-token handling semantics.
- API-053 current slice:
- `cargo check -p rustfs --lib`: passed.
- `cargo check --tests -p rustfs`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- Rust risk scan: passed; only existing import and path rewrites were
reviewed, with no new unwrap/expect, panic/todo/unsafe, risky casts,
ad-hoc error construction, or sensitive-token handling semantics.
- `make pre-commit`: passed.
- API-052 current slice:
- `cargo check -p rustfs --lib`: passed.
- `cargo check --tests -p rustfs`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- Rust risk scan: passed; only existing-semantic path replacement hits were
reviewed, with no new unwrap/expect, panic/todo/unsafe, risky casts,
ad-hoc error construction, or sensitive-token handling semantics.
- `make pre-commit`: passed, including 6250 nextest tests and doctests.
- API-050 current slice: - API-050 current slice:
- `cargo test -p rustfs-storage-api lifecycle_helper_defaults_preserve_existing_contracts --no-fail-fast`: - `cargo test -p rustfs-storage-api lifecycle_helper_defaults_preserve_existing_contracts --no-fail-fast`:
passed. passed.
@@ -1598,6 +1789,24 @@ Passed before push:
lines. lines.
- `make pre-commit`: passed. - `make pre-commit`: passed.
- API-051 current slice:
- `cargo check --tests -p e2e_test -p rustfs-heal -p rustfs-scanner`:
passed.
- `cargo check --manifest-path fuzz/Cargo.toml --all-targets`: passed.
- `cargo test -p rustfs-heal --test endpoint_index_test test_endpoint_index_settings --no-fail-fast`:
passed.
- `cargo test -p rustfs-scanner --test lifecycle_integration_test --no-run`:
passed.
- `cargo test -p e2e_test --no-run`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- Rust risk scan: passed; only existing test `unwrap` calls were touched by
import path rewrites, with no new unwrap/expect, panic/todo/unsafe, risky
casts, ad-hoc error construction, or sensitive-token handling semantics.
- `make pre-commit`: passed.
- S-015 current slice: - S-015 current slice:
- `cargo test -p rustfs-policy test_legacy_kms_admin_actions_are_rejected --no-fail-fast`: - `cargo test -p rustfs-policy test_legacy_kms_admin_actions_are_rejected --no-fail-fast`:
passed. passed.
@@ -0,0 +1,97 @@
# Placement And Repair Invariants
This inventory covers `G-012` for `rustfs/backlog#666`. It records the current
object placement, readiness, lock quorum, scanner, and repair boundaries that
later scheduler or topology work must preserve.
## Object To Set Hash Rule
Objects reach a set through `Sets::get_disks_by_key`, which calls
`get_hashed_set_index` on the object key:
- `DistributionAlgoVersion::V1` uses `crc_hash(input, set_count)`.
- `DistributionAlgoVersion::V2` and `V3` use
`sip_hash(input, set_count, format_id_bytes)`.
- The format ID is part of the V2/V3 distribution seed, so changing the seed,
object key, set count, or algorithm changes placement.
Preservation rule: every object read, write, list, heal, repair, and
decommission path that resolves a set for an existing object must preserve the
same object key and format distribution algorithm.
## Pool, Set, And Disk Assignment Boundary
Pool selection is separate from set hashing:
- Existing objects are discovered across pools and resolved to the best current
pool candidate before reads or updates continue.
- New object writes select an available pool from current per-pool free-space
inputs after suspended or rebalancing pools are skipped.
- Set selection inside a pool still uses the object-to-set hash rule above.
- Disk index assignment comes from endpoint and format metadata, not from a
scheduler decision.
Boundary rule: schedulers may influence admission, worker concurrency, or
buffer sizing, but they must not rewrite pool, set, or disk indexes.
## Readiness And Lock Quorum Boundary
Runtime readiness currently checks storage and lock health independently:
- Storage readiness requires every observed set to meet write quorum based on
the set drive count and storage class data/parity shape.
- Lock readiness aggregates per-set lock-client host quorum and fails fast if
any set loses quorum.
- Object and bucket mutations acquire namespace locks through the existing
storage lock wrappers before changing object or bucket state.
Boundary rule: readiness and lock quorum must stay set-aware. A global healthy
disk count or global connected-host count is not sufficient when any individual
set is below quorum.
## Scanner Budget Preservation
Scanner cycles are bounded by `ScannerCycleBudget`:
- Runtime budget cancels the child token after the configured duration.
- Object budget cancels after the configured object count.
- Directory budget rejects additional directories and cancels with the
directories reason.
- Partial-cycle metrics and checkpoints use the budget reason.
Preservation rule: later scheduler work can change how scan cycles are admitted
only if it preserves the budget reason, checkpoint reason, and child-token
cancellation behavior.
## Heal Admission Preservation
Scanner and background repair work enter the heal manager through explicit
admission:
- Scanner object heal requests are low priority and may be accepted, merged,
rejected as full, or dropped.
- Required/high-priority heal candidates escalate on non-admission instead of
silently disappearing.
- Heal queue admission deduplicates queued and active work unless the request
explicitly forces admission.
- Full queues can drop low-priority work or displace lower-priority work for a
higher-priority request according to current manager rules.
Preservation rule: repair scheduling changes must keep admission outcomes
observable and must not convert rejected or dropped repair work into silent
success.
## Behavior Change Gates
Any later placement or repair PR must use the following gates:
- Placement gate: prove object-to-set hashing is unchanged for existing object
keys and format algorithms.
- Pool gate: prove pool selection does not choose suspended or rebalancing
pools unless the existing path already allows it.
- Quorum gate: prove storage readiness and lock readiness remain per-set.
- Scanner gate: prove scan budget reason and checkpoint mapping remain stable.
- Heal gate: prove low-priority scanner heal, forced heal, duplicate merge, and
queue-full outcomes remain distinct.
- Rollback gate: if a new scheduler sidecar is disabled, placement and repair
must fall back to the current direct ECStore/scanner/heal behavior.
@@ -0,0 +1,81 @@
# Profiling And NUMA Capability Inventory
This inventory covers `G-013` for `rustfs/backlog#667`. It records the current
profiling, memory sampling, allocator, and NUMA baseline before optional runtime
sidecars are designed.
## Platform Support Matrix
| Capability | Current support | Current owner | Baseline recommendation |
|---|---|---|---|
| CPU pprof dump | Linux and macOS builds use `pprof`; other targets return an unsupported-platform error. | `rustfs/src/profiling.rs` | Keep CPU profiling opt-in through existing env flags and cancellation token. |
| Continuous CPU profiling | Linux and macOS builds can hold a continuous `ProfilerGuard` when enabled. | `rustfs/src/profiling.rs` | Preserve single-guard ownership and avoid starting multiple continuous guards. |
| Periodic CPU profiling | Linux and macOS builds can spawn a periodic sampling loop. | `rustfs/src/profiling.rs` | Keep the loop cancellation-driven and non-fatal. |
| Jemalloc memory pprof | Only `linux` + `gnu` + `x86_64` exposes jemalloc pprof dumping. Other supported builds return an unsupported-target error. | `rustfs/src/profiling.rs` | Treat memory pprof as optional and target-gated. |
| Periodic memory pprof | Only runs where jemalloc profiling control is available and active. | `rustfs/src/profiling.rs` | Keep inactive jemalloc as a skipped dump, not a startup failure. |
| Process/system memory sampling | Uses `rustfs_io_metrics::snapshot_process_resource_and_system` plus `sysinfo` total memory. | `rustfs/src/memory_observability.rs` | Keep sampling portable and metric-gated. |
| cgroup memory sampling | Reads Linux cgroup v2 or v1 memory files when present. Missing files produce no cgroup split. | `rustfs/src/memory_observability.rs` | Keep cgroup data opportunistic and absent-safe. |
| Allocator reclaim | Uses jemalloc backend on `linux` + `gnu` + `x86_64`; otherwise mimalloc variants. | `rustfs/src/allocator_reclaim.rs` | Keep backend detection read-only and preserve effective-force behavior. |
| eBPF | No runtime eBPF sidecar is currently wired into startup. | N/A | Treat eBPF as future optional Linux-only inventory, never as a required baseline. |
| NUMA | No NUMA placement or topology controller is currently wired into startup. | N/A | Treat NUMA as future optional capability with no-op fallback. |
## Cross-Platform Baseline
The current safe baseline is:
- Profiling is opt-in through env flags and must not make startup fatal.
- Unsupported profiling targets return structured unsupported errors or skip
startup tasks.
- Memory observability records process/system metrics and adds cgroup split
only when cgroup files exist.
- Allocator reclaim observes active HTTP, delete-tail, scanner, heal, erasure,
and GET-buffer activity before reclaiming.
- Runtime thread sizing remains owned by the Tokio runtime builder and sysinfo
core detection, not NUMA topology.
## Optional Sidecar Invariants
Future sidecars for profiling, eBPF, or NUMA must preserve these invariants:
- Sidecars must be disabled by default or target-gated until explicitly enabled.
- Unsupported targets must degrade to no-op status, not panic or fail startup.
- Sidecars must use the runtime cancellation token or an equivalent explicit
shutdown handle.
- Sidecars must not mutate Tokio worker counts after runtime creation.
- Profiling output directory fallback must stay local to profiling and must not
affect object storage paths.
- NUMA fallback must preserve current runtime thread defaults, storage set
placement, and request admission behavior.
## First Implementation Candidates
`API-013`:
- Define a read-only capability contract for profiling, cgroup memory, eBPF,
allocator backend, and NUMA availability.
- Keep the contract in a low-dependency crate and report unsupported states
explicitly.
`R-016`:
- Wire storage runtime startup to consume capability snapshots read-only.
- Do not start sidecars or mutate runtime worker ownership in the same PR.
`X-012`:
- Add an optional profiling/eBPF extension point that can observe capability
status.
- Keep unsupported targets and disabled sidecars as no-op.
`X-013`:
- Add extension tests for disabled, unsupported, and enabled capability
snapshots.
- Verify no startup fatal boundary is added for optional profiling sidecars.
`R-017`:
- If a runtime service sidecar is added later, start it from the startup service
boundary with explicit shutdown ownership.
- Preserve current service order, KMS/audit/notification fatal boundaries, and
scanner/heal startup semantics.
@@ -0,0 +1,44 @@
# Runtime Capability Contracts
This document records the `rustfs/backlog#660` PR-08 and PR-09 contract slice.
It adds read-only observability and topology snapshot shapes to
`rustfs-storage-api` without coupling the contract crate to runtime, ECStore,
admin routes, profiling, or observability implementation crates.
## Observability Snapshot Contract
`ObservabilitySnapshot` records:
- Runtime telemetry capability state.
- Userspace CPU and memory profiling capability state.
- Process, system, and cgroup memory sampling state.
- Platform support for target triple, OS, architecture, allocator, eBPF, and
NUMA capability.
Unsupported, disabled, and unknown states are represented by `CapabilityState`
instead of failing snapshot construction. The contract is intentionally read
only and does not replace existing profiling routes, telemetry APIs, exporter
pipelines, or startup behavior.
## Topology Snapshot Contract
`TopologySnapshot` records:
- Pool, set, and disk identity indexes plus optional stable IDs.
- Optional zone, rack, node, media, NUMA, and additional labels.
- Topology-wide profiling, NUMA, failure-domain label, and media-label
capability states.
- Per-disk media, failure-domain, NUMA, and profiling capability states.
Missing labels are represented as absent `Option` values. Extra topology labels
belong in the `additional` label map, so future inventory labels do not require
ECStore type leakage.
## Boundary Rules
- No `rustfs-ecstore`, `rustfs-obs`, Axum, KMS, admin route, OTEL, eBPF, or
profiling implementation dependency is added to `rustfs-storage-api`.
- No placement, membership, NUMA pinning, profiling, startup, admin route, or
exporter behavior changes are part of this contract slice.
- Providers must map implementation failures into `CapabilitySnapshotError`
before crossing the contract boundary.
+78
View File
@@ -0,0 +1,78 @@
# Scheduler Baseline Inventory
This inventory covers `G-011` for `rustfs/backlog#675`. It is a docs-only
snapshot of the current scheduling, backpressure, worker, scanner, heal, and
runtime-builder ownership. It does not define new behavior.
## Current Owners
| Surface | Current owner | Current responsibility | Migration boundary |
|---|---|---|---|
| `ConcurrencyManager` | `rustfs/src/storage/concurrency/manager.rs` | Owns the RustFS S3 read-path disk-read semaphore, I/O metrics, priority queue, storage media detection, access-pattern detection, and buffer strategy. | Keep request admission and I/O metrics behavior stable until a controller can consume the same state explicitly. |
| `SchedulerManager` | `crates/concurrency/src/scheduler.rs` | Provides a reusable facade over `rustfs-io-core::IoScheduler` and derives buffer/priority decisions from `SchedulerPolicy`. | Treat it as reusable library surface; do not assume the RustFS S3 read path has already moved to this facade. |
| `BackpressureManager` | `crates/concurrency/src/backpressure.rs` | Provides a reusable duplex-pipe backpressure facade over `rustfs-io-core::BackpressureMonitor`. | Keep pipe sizing and watermark policy separate from object-read disk semaphore admission. |
| RustFS backpressure monitor | `rustfs/src/storage/backpressure.rs` | Tracks object-pipe watermark state used by RustFS storage backpressure tests and helpers. | Preserve current state labels and watermark semantics when consolidating with reusable facades. |
| `Workers` | `crates/concurrency/src/workers.rs` | Provides cooperative worker-slot admission with `take`, `give`, and `wait`; current background workflows use it for bounded set workers. | Preserve blocking/wakeup semantics and over-release clamping. |
| Scanner cycle budget | `crates/scanner/src/scanner_budget.rs` | Cancels a child token when runtime, object-count, or directory-count budget is reached. | Preserve partial-cycle reason mapping and checkpoint accounting. |
| Heal admission | `crates/heal/src/heal/manager.rs`, `crates/heal/src/heal/channel.rs`, `rustfs_common::heal_channel` | Owns priority queue admission, duplicate merge/drop/full results, active-task tracking, retry admission, and channel responses. | Preserve low-priority scanner behavior and high-priority escalation gates. |
| Tokio runtime builder | `rustfs/src/server/runtime.rs` | Builds the multi-thread runtime from env/defaults, sets thread counts, stack, queue/event intervals, I/O event cap, thread name, and optional dial9 tracing. | Keep runtime defaults and env names stable when later startup phases move ownership. |
## Current Flow
```mermaid
flowchart TD
http["HTTP/S3 request"] --> app["app object usecase"]
app --> guard["ConcurrencyManager::track_request"]
app --> permit["disk-read semaphore permit"]
permit --> strategy["I/O queue status and buffer strategy"]
strategy --> ecstore["ECStore object/read path"]
ecstore --> setdisks["hashed set disks"]
scanner["scanner cycle"] --> budget["ScannerCycleBudget"]
budget --> folder["folder/object scan"]
folder --> healreq["heal channel request"]
healreq --> admission["HealManager admission queue"]
admission --> healworker["heal workers and retries"]
startup["startup entrypoint"] --> runtime["Tokio runtime builder"]
runtime --> services["background services"]
```
## Missing State For Later Work
`R-015` storage foundation:
- Needs a stable inventory of endpoint publication, local disk prewarm, lock
client setup, and per-set readiness state before any scheduler/controller
consumes storage topology.
- Must not infer set availability only from request-path I/O metrics.
`E-011` extension/runtime consumers:
- Need explicit ownership for runtime admission snapshots before extensions can
observe scheduler or backpressure state.
- Must not receive mutable handles to `ConcurrencyManager`, heal queues, or
scanner budget tokens.
`C-011` controller work:
- Needs desired/current/status snapshots for request admission, scanner budget,
and heal queue pressure before any controller can reconcile them.
- Must keep worker mutation explicit. Read-only status should report `None` or
no-op mutation until a reviewed worker lifecycle PR exists.
## Preservation Invariants
- Request reads must keep the same disk-read semaphore admission and active GET
accounting.
- I/O queue status and congestion metrics must remain derived from the same
permit counts.
- Scanner budget cancellation must keep its reason as runtime, objects, or
directories.
- Scanner inline-heal compatibility must continue to use asynchronous heal
admission.
- Heal duplicate admission must prefer merge semantics before full-queue
rejection.
- High-priority heal admission must still be able to displace lower-priority
queued work where the current manager allows it.
- Tokio runtime env names and fallback defaults must remain unchanged.
+1 -1
View File
@@ -4,7 +4,7 @@
mod storage_compat; mod storage_compat;
use libfuzzer_sys::fuzz_target; use libfuzzer_sys::fuzz_target;
use crate::storage_compat::ecstore::bucket::utils::{ use crate::storage_compat::{
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname, check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
}; };
@@ -12,12 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
pub(crate) mod ecstore { pub(crate) use rustfs_ecstore::bucket::utils::{
pub(crate) mod bucket { check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
pub(crate) mod utils { };
pub(crate) use rustfs_ecstore::bucket::utils::{
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
};
}
}
}
+1 -1
View File
@@ -4,7 +4,7 @@
mod storage_compat; mod storage_compat;
use libfuzzer_sys::fuzz_target; use libfuzzer_sys::fuzz_target;
use crate::storage_compat::ecstore::bucket::utils::{check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix}; use crate::storage_compat::{check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix};
use rustfs_utils::path::{clean, path_join}; use rustfs_utils::path::{clean, path_join};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
@@ -12,12 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
pub(crate) mod ecstore { pub(crate) use rustfs_ecstore::bucket::utils::{
pub(crate) mod bucket { check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix,
pub(crate) mod utils { };
pub(crate) use rustfs_ecstore::bucket::utils::{
check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix,
};
}
}
}
+1 -1
View File
@@ -14,7 +14,7 @@
use crate::admin::auth::authenticate_request; use crate::admin::auth::authenticate_request;
use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::bucket::versioning_sys::BucketVersioningSys; use crate::admin::storage_compat::versioning_sys::BucketVersioningSys;
use crate::app::context::resolve_object_store_handle; use crate::app::context::resolve_object_store_handle;
use crate::auth::get_condition_values; use crate::auth::get_condition_values;
use crate::server::{ADMIN_PREFIX, RemoteAddr}; use crate::server::{ADMIN_PREFIX, RemoteAddr};
@@ -24,7 +24,7 @@ pub(crate) async fn load_server_config_from_store() -> S3Result<Config> {
return Ok(Config::new()); return Ok(Config::new());
}; };
crate::admin::storage_compat::ecstore::config::com::read_config_without_migrate(store) crate::admin::storage_compat::com::read_config_without_migrate(store)
.await .await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e)) .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))
} }
@@ -82,7 +82,7 @@ where
return Err(s3_error!(InternalError, "server storage not initialized")); return Err(s3_error!(InternalError, "server storage not initialized"));
}; };
let mut config = crate::admin::storage_compat::ecstore::config::com::read_config_without_migrate(store.clone()) let mut config = crate::admin::storage_compat::com::read_config_without_migrate(store.clone())
.await .await
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?; .map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
@@ -90,7 +90,7 @@ where
return Ok(()); return Ok(());
} }
crate::admin::storage_compat::ecstore::config::com::save_server_config(store, &config) crate::admin::storage_compat::com::save_server_config(store, &config)
.await .await
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?; .map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
+10 -12
View File
@@ -12,19 +12,17 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::admin::storage_compat::ecstore::bucket::utils::{deserialize, serialize}; use crate::admin::storage_compat::utils::{deserialize, serialize};
use crate::admin::storage_compat::ecstore::{ use crate::admin::storage_compat::{
bucket::{ StorageError,
metadata::{ metadata::{
BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG,
BucketMetadata, OBJECT_LOCK_CONFIG, BucketMetadata, OBJECT_LOCK_CONFIG,
},
metadata_sys,
quota::BucketQuota,
target::BucketTargets,
}, },
error::StorageError, metadata_sys,
quota::BucketQuota,
target::BucketTargets,
}; };
use crate::{ use crate::{
admin::{ admin::{
+14 -14
View File
@@ -18,12 +18,12 @@ use crate::admin::service::config::{
apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem, signal_config_snapshot_reload, signal_dynamic_config_reload, apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem, signal_config_snapshot_reload, signal_dynamic_config_reload,
validate_server_config, validate_server_config,
}; };
use crate::admin::storage_compat::ecstore::config::com::STORAGE_CLASS_SUB_SYS; use crate::admin::storage_compat::RUSTFS_META_BUCKET;
use crate::admin::storage_compat::ecstore::config::com::{ use crate::admin::storage_compat::com::STORAGE_CLASS_SUB_SYS;
use crate::admin::storage_compat::com::{
delete_config, read_config, read_config_without_migrate, save_config, save_server_config, delete_config, read_config, read_config_without_migrate, save_config, save_server_config,
}; };
use crate::admin::storage_compat::ecstore::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV}; use crate::admin::storage_compat::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
use crate::admin::storage_compat::ecstore::disk::RUSTFS_META_BUCKET;
use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body}; use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body};
use crate::app::context::resolve_object_store_handle; use crate::app::context::resolve_object_store_handle;
use crate::auth::{check_key_valid, get_session_token}; use crate::auth::{check_key_valid, get_session_token};
@@ -711,7 +711,7 @@ fn success_response(config_applied: bool) -> S3Result<S3Response<(StatusCode, Bo
Ok(S3Response::with_headers((StatusCode::OK, Body::default()), headers)) Ok(S3Response::with_headers((StatusCode::OK, Body::default()), headers))
} }
fn object_store() -> S3Result<std::sync::Arc<crate::admin::storage_compat::ecstore::store::ECStore>> { fn object_store() -> S3Result<std::sync::Arc<crate::admin::storage_compat::ECStore>> {
resolve_object_store_handle().ok_or_else(|| s3_error!(InternalError, "server storage not initialized")) resolve_object_store_handle().ok_or_else(|| s3_error!(InternalError, "server storage not initialized"))
} }
@@ -756,7 +756,7 @@ fn config_update_sub_system(directives: &[ConfigDirective]) -> S3Result<Option<&
fn validate_config_directives(directives: &[ConfigDirective]) -> S3Result<()> { fn validate_config_directives(directives: &[ConfigDirective]) -> S3Result<()> {
if DEFAULT_KVS.get().is_none() { if DEFAULT_KVS.get().is_none() {
crate::admin::storage_compat::ecstore::config::init(); crate::admin::storage_compat::init();
} }
let Some(defaults) = DEFAULT_KVS.get() else { let Some(defaults) = DEFAULT_KVS.get() else {
return Err(s3_error!(InternalError, "config defaults are not initialized")); return Err(s3_error!(InternalError, "config defaults are not initialized"));
@@ -1410,7 +1410,7 @@ fn env_help_key(sub_system: &str, key: &str) -> String {
fn default_help_postfix(sub_system: &str, key: &str) -> String { fn default_help_postfix(sub_system: &str, key: &str) -> String {
if DEFAULT_KVS.get().is_none() { if DEFAULT_KVS.get().is_none() {
crate::admin::storage_compat::ecstore::config::init(); crate::admin::storage_compat::init();
} }
DEFAULT_KVS DEFAULT_KVS
@@ -1897,7 +1897,7 @@ mod tests {
#[test] #[test]
fn full_config_export_can_be_reapplied() { fn full_config_export_can_be_reapplied() {
crate::admin::storage_compat::ecstore::config::init(); crate::admin::storage_compat::init();
let mut original = ServerConfig::new(); let mut original = ServerConfig::new();
apply_set_directives( apply_set_directives(
&mut original, &mut original,
@@ -1944,7 +1944,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
#[test] #[test]
fn build_help_response_appends_default_value_postfix() { fn build_help_response_appends_default_value_postfix() {
crate::admin::storage_compat::ecstore::config::init(); crate::admin::storage_compat::init();
let response = build_help_response(Some("identity_openid"), Some("scopes"), false).expect("help response"); let response = build_help_response(Some("identity_openid"), Some("scopes"), false).expect("help response");
assert_eq!(response.keys_help.len(), 2); assert_eq!(response.keys_help.len(), 2);
@@ -2057,7 +2057,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
#[test] #[test]
fn render_selected_config_includes_env_override_lines() { fn render_selected_config_includes_env_override_lines() {
crate::admin::storage_compat::ecstore::config::init(); crate::admin::storage_compat::init();
temp_env::with_vars( temp_env::with_vars(
[ [
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example")), ("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example")),
@@ -2093,7 +2093,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
#[test] #[test]
fn render_selected_config_lists_env_only_targets() { fn render_selected_config_lists_env_only_targets() {
crate::admin::storage_compat::ecstore::config::init(); crate::admin::storage_compat::init();
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || { temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || {
let config = ServerConfig::new(); let config = ServerConfig::new();
let rendered = String::from_utf8( let rendered = String::from_utf8(
@@ -2116,7 +2116,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
#[test] #[test]
fn render_selected_config_supports_specific_env_only_target_queries() { fn render_selected_config_supports_specific_env_only_target_queries() {
crate::admin::storage_compat::ecstore::config::init(); crate::admin::storage_compat::init();
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || { temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || {
let config = ServerConfig::new(); let config = ServerConfig::new();
let rendered = String::from_utf8( let rendered = String::from_utf8(
@@ -2139,7 +2139,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
#[test] #[test]
fn render_selected_config_orders_default_before_named_targets() { fn render_selected_config_orders_default_before_named_targets() {
crate::admin::storage_compat::ecstore::config::init(); crate::admin::storage_compat::init();
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ALPHA", Some("http://alpha.example"))], || { temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ALPHA", Some("http://alpha.example"))], || {
let mut config = ServerConfig::new(); let mut config = ServerConfig::new();
apply_set_directives( apply_set_directives(
@@ -2320,7 +2320,7 @@ identity_openid client_id="existing-client""#,
#[test] #[test]
fn storage_class_get_target_none_matches_full_export() { fn storage_class_get_target_none_matches_full_export() {
crate::admin::storage_compat::ecstore::config::init(); crate::admin::storage_compat::init();
let mut config = ServerConfig::new(); let mut config = ServerConfig::new();
apply_set_directives( apply_set_directives(
&mut config, &mut config,
+5 -5
View File
@@ -14,8 +14,8 @@
use crate::admin::auth::{authenticate_request, validate_admin_request}; use crate::admin::auth::{authenticate_request, validate_admin_request};
use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::bucket::utils::is_valid_object_prefix; use crate::admin::storage_compat::is_reserved_or_invalid_bucket;
use crate::admin::storage_compat::ecstore::store_utils::is_reserved_or_invalid_bucket; use crate::admin::storage_compat::utils::is_valid_object_prefix;
use crate::app::context::resolve_object_store_handle; use crate::app::context::resolve_object_store_handle;
use crate::server::ADMIN_PREFIX; use crate::server::ADMIN_PREFIX;
use crate::server::RemoteAddr; use crate::server::RemoteAddr;
@@ -345,10 +345,10 @@ fn should_handle_root_heal_directly(hip: &HealInitParams) -> bool {
hip.bucket.is_empty() && hip.obj_prefix.is_empty() && hip.client_token.is_empty() && !hip.force_stop hip.bucket.is_empty() && hip.obj_prefix.is_empty() && hip.client_token.is_empty() && !hip.force_stop
} }
fn map_root_heal_status(heal_err: Option<crate::admin::storage_compat::ecstore::error::Error>) -> S3Result<()> { fn map_root_heal_status(heal_err: Option<crate::admin::storage_compat::Error>) -> S3Result<()> {
match heal_err { match heal_err {
None => Ok(()), None => Ok(()),
Some(crate::admin::storage_compat::ecstore::error::StorageError::NoHealRequired) => { Some(crate::admin::storage_compat::StorageError::NoHealRequired) => {
info!( info!(
event = EVENT_ADMIN_RESPONSE_EMITTED, event = EVENT_ADMIN_RESPONSE_EMITTED,
component = LOG_COMPONENT_ADMIN_API, component = LOG_COMPONENT_ADMIN_API,
@@ -703,7 +703,7 @@ mod tests {
encode_heal_task_status, heal_channel_response_items, heal_channel_response_summary, json_response, map_heal_response, encode_heal_task_status, heal_channel_response_items, heal_channel_response_summary, json_response, map_heal_response,
map_root_heal_status, should_handle_root_heal_directly, validate_heal_request_mode, validate_heal_target, map_root_heal_status, should_handle_root_heal_directly, validate_heal_request_mode, validate_heal_target,
}; };
use crate::admin::storage_compat::ecstore::error::StorageError; use crate::admin::storage_compat::StorageError;
use bytes::Bytes; use bytes::Bytes;
use http::StatusCode; use http::StatusCode;
use http::Uri; use http::Uri;
+1 -1
View File
@@ -16,7 +16,7 @@
use crate::admin::auth::validate_admin_request; use crate::admin::auth::validate_admin_request;
use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::config::com::{read_config, save_config}; use crate::admin::storage_compat::com::{read_config, save_config};
use crate::app::context::{resolve_kms_runtime_service_manager, resolve_object_store_handle}; use crate::app::context::{resolve_kms_runtime_service_manager, resolve_object_store_handle};
use crate::auth::{check_key_valid, get_session_token}; use crate::auth::{check_key_valid, get_session_token};
use crate::server::{ADMIN_PREFIX, RemoteAddr}; use crate::server::{ADMIN_PREFIX, RemoteAddr};
+1 -1
View File
@@ -20,7 +20,7 @@
use crate::admin::auth::validate_admin_request; use crate::admin::auth::validate_admin_request;
use crate::admin::router::Operation; use crate::admin::router::Operation;
use crate::admin::storage_compat::ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics}; use crate::admin::storage_compat::{CollectMetricsOpts, MetricType, collect_local_metrics};
use crate::auth::{check_key_valid, get_session_token}; use crate::auth::{check_key_valid, get_session_token};
use crate::server::RemoteAddr; use crate::server::RemoteAddr;
use crate::storage::request_context::spawn_traced; use crate::storage::request_context::spawn_traced;
@@ -13,7 +13,7 @@
// limitations under the License. // limitations under the License.
use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, Operation, S3Router}; use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::global::get_global_region; use crate::admin::storage_compat::get_global_region;
use crate::app::context::resolve_object_store_handle; use crate::app::context::resolve_object_store_handle;
use crate::auth::{check_key_valid, get_session_token}; use crate::auth::{check_key_valid, get_session_token};
use crate::error::ApiError; use crate::error::ApiError;
@@ -649,7 +649,7 @@ async fn preflight_zip_items(request: &CreateObjectZipDownloadRequest, items: &[
Ok(()) Ok(())
} }
fn storage_error_to_s3(err: crate::admin::storage_compat::ecstore::error::Error) -> s3s::S3Error { fn storage_error_to_s3(err: crate::admin::storage_compat::Error) -> s3s::S3Error {
ApiError::from(err).into() ApiError::from(err).into()
} }
+1 -1
View File
@@ -15,7 +15,7 @@
use super::sts::create_oidc_sts_credentials; use super::sts::create_oidc_sts_credentials;
use crate::admin::auth::validate_admin_request; use crate::admin::auth::validate_admin_request;
use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::config::com::{read_config_without_migrate, save_server_config}; use crate::admin::storage_compat::com::{read_config_without_migrate, save_server_config};
use crate::app::context::resolve_object_store_handle; use crate::app::context::resolve_object_store_handle;
use crate::auth::{check_key_valid, get_session_token}; use crate::auth::{check_key_valid, get_session_token};
use crate::server::{ADMIN_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr}; use crate::server::{ADMIN_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr};
+1 -1
View File
@@ -99,7 +99,7 @@ macro_rules! log_pool_response_emitted {
}; };
} }
fn endpoints_from_context() -> Option<crate::admin::storage_compat::ecstore::endpoints::EndpointServerPools> { fn endpoints_from_context() -> Option<crate::admin::storage_compat::EndpointServerPools> {
resolve_endpoints_handle() resolve_endpoints_handle()
} }
+4 -4
View File
@@ -16,9 +16,9 @@
use crate::admin::auth::{validate_admin_request, validate_admin_request_with_bucket}; use crate::admin::auth::{validate_admin_request, validate_admin_request_with_bucket};
use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::bucket::metadata_sys::BucketMetadataSys; use crate::admin::storage_compat::metadata_sys::BucketMetadataSys;
use crate::admin::storage_compat::ecstore::bucket::quota::checker::QuotaChecker; use crate::admin::storage_compat::quota::checker::QuotaChecker;
use crate::admin::storage_compat::ecstore::bucket::quota::{BucketQuota, QuotaError, QuotaOperation}; use crate::admin::storage_compat::quota::{BucketQuota, QuotaError, QuotaOperation};
use crate::app::context::{resolve_bucket_metadata_handle, resolve_object_store_handle}; use crate::app::context::{resolve_bucket_metadata_handle, resolve_object_store_handle};
use crate::auth::{check_key_valid, get_session_token}; use crate::auth::{check_key_valid, get_session_token};
use crate::server::ADMIN_PREFIX; use crate::server::ADMIN_PREFIX;
@@ -175,7 +175,7 @@ async fn current_usage_from_context(bucket: &str) -> u64 {
return 0; return 0;
}; };
match crate::admin::storage_compat::ecstore::data_usage::load_data_usage_from_backend(store).await { match crate::admin::storage_compat::load_data_usage_from_backend(store).await {
Ok(data_usage_info) => data_usage_info Ok(data_usage_info) => data_usage_info
.buckets_usage .buckets_usage
.get(bucket) .get(bucket)
+5 -9
View File
@@ -12,10 +12,8 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::admin::storage_compat::ecstore::{ use crate::admin::storage_compat::{
error::StorageError, DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, StorageError, get_global_notification_sys,
notification_sys::get_global_notification_sys,
rebalance::{DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta},
}; };
use crate::{ use crate::{
admin::{ admin::{
@@ -150,7 +148,7 @@ fn build_rebalance_pool_progress(
now: OffsetDateTime, now: OffsetDateTime,
stop_time: Option<OffsetDateTime>, stop_time: Option<OffsetDateTime>,
percent_free_goal: f64, percent_free_goal: f64,
ps: &crate::admin::storage_compat::ecstore::rebalance::RebalanceStats, ps: &crate::admin::storage_compat::RebalanceStats,
) -> Option<RebalPoolProgress> { ) -> Option<RebalPoolProgress> {
let total_bytes_to_rebal = ps.init_capacity as f64 * percent_free_goal - ps.init_free_space as f64; let total_bytes_to_rebal = ps.init_capacity as f64 * percent_free_goal - ps.init_free_space as f64;
let terminal_time = ps.info.end_time.or(stop_time); let terminal_time = ps.info.end_time.or(stop_time);
@@ -193,7 +191,7 @@ fn build_rebalance_pool_statuses(
now: OffsetDateTime, now: OffsetDateTime,
stop_time: Option<OffsetDateTime>, stop_time: Option<OffsetDateTime>,
percent_free_goal: f64, percent_free_goal: f64,
pool_stats: &[crate::admin::storage_compat::ecstore::rebalance::RebalanceStats], pool_stats: &[crate::admin::storage_compat::RebalanceStats],
disk_stats: &[DiskStat], disk_stats: &[DiskStat],
) -> Vec<RebalancePoolStatus> { ) -> Vec<RebalancePoolStatus> {
pool_stats pool_stats
@@ -563,9 +561,7 @@ mod rebalance_handler_tests {
RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, build_rebalance_pool_statuses, rebalance_pool_used, RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, build_rebalance_pool_statuses, rebalance_pool_used,
rebalance_remaining_buckets, rebalance_used_pct, rebalance_remaining_buckets, rebalance_used_pct,
}; };
use crate::admin::storage_compat::ecstore::rebalance::{ use crate::admin::storage_compat::{DiskStat, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceStats};
DiskStat, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceStats,
};
use time::OffsetDateTime; use time::OffsetDateTime;
#[test] #[test]
+9 -9
View File
@@ -15,15 +15,15 @@
use crate::admin::auth::validate_admin_request; use crate::admin::auth::validate_admin_request;
use crate::admin::handlers::site_replication::site_replication_peer_deployment_id_for_endpoint; use crate::admin::handlers::site_replication::site_replication_peer_deployment_id_for_endpoint;
use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::{BucketTargetError, BucketTargetSys}; use crate::admin::storage_compat::StorageError;
use crate::admin::storage_compat::ecstore::bucket::metadata::BUCKET_TARGETS_FILE; use crate::admin::storage_compat::bucket_target_sys::{BucketTargetError, BucketTargetSys};
use crate::admin::storage_compat::ecstore::bucket::metadata_sys; use crate::admin::storage_compat::global_rustfs_port;
use crate::admin::storage_compat::ecstore::bucket::metadata_sys::get_replication_config; use crate::admin::storage_compat::metadata::BUCKET_TARGETS_FILE;
use crate::admin::storage_compat::ecstore::bucket::replication::BucketStats; use crate::admin::storage_compat::metadata_sys;
use crate::admin::storage_compat::ecstore::bucket::replication::GLOBAL_REPLICATION_STATS; use crate::admin::storage_compat::metadata_sys::get_replication_config;
use crate::admin::storage_compat::ecstore::bucket::target::BucketTarget; use crate::admin::storage_compat::replication::BucketStats;
use crate::admin::storage_compat::ecstore::error::StorageError; use crate::admin::storage_compat::replication::GLOBAL_REPLICATION_STATS;
use crate::admin::storage_compat::ecstore::global::global_rustfs_port; use crate::admin::storage_compat::target::BucketTarget;
use crate::admin::utils::read_compatible_admin_body; use crate::admin::utils::read_compatible_admin_body;
use crate::app::context::resolve_object_store_handle; use crate::app::context::resolve_object_store_handle;
use crate::auth::{check_key_valid, get_session_token}; use crate::auth::{check_key_valid, get_session_token};
+15 -19
View File
@@ -18,24 +18,20 @@ use crate::admin::site_replication_identity::{
canonical_endpoint, deployment_id_for_endpoint, normalize_peer_map_by_identity_with, same_identity_endpoint, canonical_endpoint, deployment_id_for_endpoint, normalize_peer_map_by_identity_with, same_identity_endpoint,
site_identity_key, site_identity_key,
}; };
use crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::BucketTargetSys; use crate::admin::storage_compat::Error as StorageError;
use crate::admin::storage_compat::ecstore::bucket::metadata::{ use crate::admin::storage_compat::bucket_target_sys::BucketTargetSys;
use crate::admin::storage_compat::com::{delete_config, read_config, save_config};
use crate::admin::storage_compat::metadata::{
BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG,
BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG,
}; };
use crate::admin::storage_compat::ecstore::bucket::metadata_sys; use crate::admin::storage_compat::metadata_sys;
use crate::admin::storage_compat::ecstore::bucket::replication::GLOBAL_REPLICATION_STATS; use crate::admin::storage_compat::replication::GLOBAL_REPLICATION_STATS;
use crate::admin::storage_compat::ecstore::bucket::replication::{ use crate::admin::storage_compat::replication::{ReplicationConfigurationExt, ResyncOpts, get_global_replication_pool};
ReplicationConfigurationExt, ResyncOpts, get_global_replication_pool, use crate::admin::storage_compat::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials};
}; use crate::admin::storage_compat::utils::{deserialize, serialize};
use crate::admin::storage_compat::ecstore::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials}; use crate::admin::storage_compat::versioning::VersioningApi;
use crate::admin::storage_compat::ecstore::bucket::utils::{deserialize, serialize}; use crate::admin::storage_compat::{get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port};
use crate::admin::storage_compat::ecstore::bucket::versioning::VersioningApi;
use crate::admin::storage_compat::ecstore::config::com::{delete_config, read_config, save_config};
use crate::admin::storage_compat::ecstore::error::Error as StorageError;
use crate::admin::storage_compat::ecstore::global::{
get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port,
};
use crate::admin::utils::{encode_compatible_admin_payload, read_compatible_admin_body}; use crate::admin::utils::{encode_compatible_admin_payload, read_compatible_admin_body};
use crate::app::context::resolve_object_store_handle; use crate::app::context::resolve_object_store_handle;
use crate::auth::{check_key_valid, get_session_token}; use crate::auth::{check_key_valid, get_session_token};
@@ -656,7 +652,7 @@ async fn site_replication_peer_client() -> S3Result<reqwest::Client> {
built built
} }
fn runtime_tls_enabled_with(endpoints: Option<&crate::admin::storage_compat::ecstore::endpoints::EndpointServerPools>) -> bool { fn runtime_tls_enabled_with(endpoints: Option<&crate::admin::storage_compat::EndpointServerPools>) -> bool {
if !rustfs_utils::get_env_str(ENV_RUSTFS_TLS_PATH, DEFAULT_RUSTFS_TLS_PATH).is_empty() { if !rustfs_utils::get_env_str(ENV_RUSTFS_TLS_PATH, DEFAULT_RUSTFS_TLS_PATH).is_empty() {
return true; return true;
} }
@@ -3357,7 +3353,7 @@ fn is_stale_update(local_updated_at: OffsetDateTime, incoming_updated_at: Option
} }
fn bucket_meta_local_updated_at( fn bucket_meta_local_updated_at(
bucket_meta: &crate::admin::storage_compat::ecstore::bucket::metadata::BucketMetadata, bucket_meta: &crate::admin::storage_compat::metadata::BucketMetadata,
config_file: &str, config_file: &str,
) -> OffsetDateTime { ) -> OffsetDateTime {
match config_file { match config_file {
@@ -4578,8 +4574,8 @@ impl Operation for SRRotateServiceAccountHandler {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::admin::storage_compat::ecstore::disk::endpoint::Endpoint; use crate::admin::storage_compat::Endpoint;
use crate::admin::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::admin::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints};
use http::{HeaderMap, HeaderValue, Uri}; use http::{HeaderMap, HeaderValue, Uri};
use rustfs_common::{get_global_outbound_tls_generation, set_global_outbound_tls_generation}; use rustfs_common::{get_global_outbound_tls_generation, set_global_outbound_tls_generation};
use rustfs_policy::policy::action::S3Action; use rustfs_policy::policy::action::S3Action;
+1 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License. // limitations under the License.
use super::is_admin::IsAdminHandler; use super::is_admin::IsAdminHandler;
use crate::admin::storage_compat::ecstore::bucket::utils::serialize; use crate::admin::storage_compat::utils::serialize;
use crate::{ use crate::{
admin::{ admin::{
handlers::site_replication::site_replication_iam_change_hook, handlers::site_replication::site_replication_iam_change_hook,
+1 -4
View File
@@ -12,10 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::admin::storage_compat::ecstore::{ use crate::admin::storage_compat::{ECStore, metadata::table_catalog_path_hash, metadata_sys};
bucket::{metadata::table_catalog_path_hash, metadata_sys},
store::ECStore,
};
use crate::admin::{ use crate::admin::{
auth::{AdminResourceScope, validate_admin_request, validate_admin_request_with_bucket_object}, auth::{AdminResourceScope, validate_admin_request, validate_admin_request_with_bucket_object},
router::{AdminOperation, Operation, S3Router}, router::{AdminOperation, Operation, S3Router},
+6 -16
View File
@@ -13,21 +13,11 @@
// limitations under the License. // limitations under the License.
#![allow(unused_variables, unused_mut, unused_must_use)] #![allow(unused_variables, unused_mut, unused_must_use)]
use crate::admin::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState; use crate::admin::storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState;
use crate::admin::storage_compat::ecstore::{ use crate::admin::storage_compat::{
bucket::lifecycle::tier_last_day_stats::DailyAllTierStats, AdminError, DailyAllTierStats, ERR_TIER_ALREADY_EXISTS, ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY,
client::admin_handler_utils::AdminError, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE,
config::storageclass, ERR_TIER_NOT_FOUND, TierConfig, TierCreds, TierType, get_global_notification_sys, storageclass,
notification_sys::get_global_notification_sys,
tier::{
tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_MISSING_CREDENTIALS},
tier_admin::TierCreds,
tier_config::{TierConfig, TierType},
tier_handlers::{
ERR_TIER_ALREADY_EXISTS, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE,
ERR_TIER_NOT_FOUND,
},
},
}; };
use crate::{ use crate::{
admin::{ admin::{
@@ -916,7 +906,7 @@ impl Operation for ClearTier {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::admin::storage_compat::ecstore::bucket::lifecycle::tier_last_day_stats::LastDayTierStats; use crate::admin::storage_compat::lifecycle::tier_last_day_stats::LastDayTierStats;
use http::Uri; use http::Uri;
use matchit::Router; use matchit::Router;
+1 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License. // limitations under the License.
use crate::admin::router::Operation; use crate::admin::router::Operation;
use crate::admin::storage_compat::ecstore::rpc::PeerRestClient; use crate::admin::storage_compat::PeerRestClient;
use crate::app::context::resolve_endpoints_handle; use crate::app::context::resolve_endpoints_handle;
use http::StatusCode; use http::StatusCode;
use hyper::Uri; use hyper::Uri;
+25 -27
View File
@@ -14,24 +14,24 @@
use crate::admin::console::{is_console_path, make_console_server}; use crate::admin::console::{is_console_path, make_console_server};
use crate::admin::handlers::oidc::is_oidc_path; use crate::admin::handlers::oidc::is_oidc_path;
use crate::admin::storage_compat::ecstore::bucket::bandwidth::monitor::BandwidthDetails; use crate::admin::storage_compat::GLOBAL_BOOT_TIME;
use crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::{ use crate::admin::storage_compat::PeerRestClient;
use crate::admin::storage_compat::bandwidth::monitor::BandwidthDetails;
use crate::admin::storage_compat::bucket_target_sys::{
BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, TargetClient, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, TargetClient,
}; };
use crate::admin::storage_compat::ecstore::bucket::metadata::BUCKET_TARGETS_FILE; use crate::admin::storage_compat::com::read_config_without_migrate;
use crate::admin::storage_compat::ecstore::bucket::metadata_sys; use crate::admin::storage_compat::get_global_notification_sys;
use crate::admin::storage_compat::ecstore::bucket::replication::{ use crate::admin::storage_compat::metadata::BUCKET_TARGETS_FILE;
use crate::admin::storage_compat::metadata_sys;
use crate::admin::storage_compat::replication::{
BucketReplicationResyncStatus, BucketStats, GLOBAL_REPLICATION_STATS, ObjectOpts, ReplicationConfigurationExt, ResyncOpts, BucketReplicationResyncStatus, BucketStats, GLOBAL_REPLICATION_STATS, ObjectOpts, ReplicationConfigurationExt, ResyncOpts,
get_global_replication_pool, get_global_replication_pool,
}; };
use crate::admin::storage_compat::ecstore::bucket::target::{BucketTarget, BucketTargetType, BucketTargets}; use crate::admin::storage_compat::target::{BucketTarget, BucketTargetType, BucketTargets};
use crate::admin::storage_compat::ecstore::bucket::versioning::VersioningApi; use crate::admin::storage_compat::versioning::VersioningApi;
use crate::admin::storage_compat::ecstore::bucket::versioning_sys::BucketVersioningSys; use crate::admin::storage_compat::versioning_sys::BucketVersioningSys;
use crate::admin::storage_compat::ecstore::config::com::read_config_without_migrate; use crate::admin::storage_compat::{get_global_bucket_monitor, get_global_deployment_id, get_global_region};
use crate::admin::storage_compat::ecstore::global::GLOBAL_BOOT_TIME;
use crate::admin::storage_compat::ecstore::global::{get_global_bucket_monitor, get_global_deployment_id, get_global_region};
use crate::admin::storage_compat::ecstore::notification_sys::get_global_notification_sys;
use crate::admin::storage_compat::ecstore::rpc::PeerRestClient;
use crate::app::context::resolve_object_store_handle; use crate::app::context::resolve_object_store_handle;
use crate::app::object_usecase::DefaultObjectUsecase; use crate::app::object_usecase::DefaultObjectUsecase;
use crate::auth::{check_key_valid, get_session_token}; use crate::auth::{check_key_valid, get_session_token};
@@ -1420,9 +1420,7 @@ async fn ensure_replication_bucket_exists(bucket: &str) -> S3Result<()> {
async fn ensure_replication_config_exists(bucket: &str) -> S3Result<()> { async fn ensure_replication_config_exists(bucket: &str) -> S3Result<()> {
match metadata_sys::get_replication_config(bucket).await { match metadata_sys::get_replication_config(bucket).await {
Ok(_) => Ok(()), Ok(_) => Ok(()),
Err(crate::admin::storage_compat::ecstore::error::StorageError::ConfigNotFound) => { Err(crate::admin::storage_compat::StorageError::ConfigNotFound) => Err(s3_error!(ReplicationConfigurationNotFoundError)),
Err(s3_error!(ReplicationConfigurationNotFoundError))
}
Err(err) => Err(ApiError::from(err).into()), Err(err) => Err(ApiError::from(err).into()),
} }
} }
@@ -1947,7 +1945,7 @@ async fn resolve_replication_target_client(bucket: &str, target: &BucketTarget)
fn build_replication_probe_put_options(now: OffsetDateTime) -> PutObjectOptions { fn build_replication_probe_put_options(now: OffsetDateTime) -> PutObjectOptions {
PutObjectOptions { PutObjectOptions {
internal: crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::AdvancedPutOptions { internal: crate::admin::storage_compat::bucket_target_sys::AdvancedPutOptions {
source_version_id: Uuid::new_v4().to_string(), source_version_id: Uuid::new_v4().to_string(),
replication_status: ReplicationStatusType::Replica, replication_status: ReplicationStatusType::Replica,
source_mtime: now, source_mtime: now,
@@ -2062,7 +2060,7 @@ async fn source_bucket_requires_object_lock(bucket: &str) -> S3Result<bool> {
.object_lock_enabled .object_lock_enabled
.as_ref() .as_ref()
.is_some_and(|state| state.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)), .is_some_and(|state| state.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)),
Err(crate::admin::storage_compat::ecstore::error::StorageError::ConfigNotFound) => Ok(false), Err(crate::admin::storage_compat::StorageError::ConfigNotFound) => Ok(false),
Err(err) => Err(ApiError::from(err).into()), Err(err) => Err(ApiError::from(err).into()),
} }
} }
@@ -2776,7 +2774,7 @@ mod tests {
#[test] #[test]
fn apply_replication_reset_to_targets_updates_matching_target() { fn apply_replication_reset_to_targets_updates_matching_target() {
let mut targets = BucketTargets { let mut targets = BucketTargets {
targets: vec![crate::admin::storage_compat::ecstore::bucket::target::BucketTarget { targets: vec![crate::admin::storage_compat::target::BucketTarget {
arn: "arn:target".to_string(), arn: "arn:target".to_string(),
..Default::default() ..Default::default()
}], }],
@@ -2798,10 +2796,10 @@ mod tests {
let mut status = BucketReplicationResyncStatus::new(); let mut status = BucketReplicationResyncStatus::new();
status.targets_map.insert( status.targets_map.insert(
"arn:z".to_string(), "arn:z".to_string(),
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus { crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
resync_id: "rid-z".to_string(), resync_id: "rid-z".to_string(),
last_update: Some(datetime!(2025-01-03 00:00 UTC)), last_update: Some(datetime!(2025-01-03 00:00 UTC)),
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncFailed, resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncFailed,
failed_count: 2, failed_count: 2,
failed_size: 4, failed_size: 4,
bucket: "bucket-z".to_string(), bucket: "bucket-z".to_string(),
@@ -2811,10 +2809,10 @@ mod tests {
); );
status.targets_map.insert( status.targets_map.insert(
"arn:a".to_string(), "arn:a".to_string(),
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus { crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
resync_id: "rid-a".to_string(), resync_id: "rid-a".to_string(),
last_update: Some(datetime!(2025-01-02 00:00 UTC)), last_update: Some(datetime!(2025-01-02 00:00 UTC)),
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncCompleted, resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncCompleted,
replicated_count: 3, replicated_count: 3,
replicated_size: 9, replicated_size: 9,
bucket: "bucket-a".to_string(), bucket: "bucket-a".to_string(),
@@ -2844,10 +2842,10 @@ mod tests {
let mut status = BucketReplicationResyncStatus::new(); let mut status = BucketReplicationResyncStatus::new();
status.targets_map.insert( status.targets_map.insert(
"arn:z".to_string(), "arn:z".to_string(),
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus { crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
resync_id: "rid-z".to_string(), resync_id: "rid-z".to_string(),
last_update: Some(datetime!(2025-02-03 00:00 UTC)), last_update: Some(datetime!(2025-02-03 00:00 UTC)),
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncFailed, resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncFailed,
failed_count: 2, failed_count: 2,
failed_size: 4, failed_size: 4,
bucket: "bucket-z".to_string(), bucket: "bucket-z".to_string(),
@@ -2857,10 +2855,10 @@ mod tests {
); );
status.targets_map.insert( status.targets_map.insert(
"arn:a".to_string(), "arn:a".to_string(),
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus { crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
resync_id: "rid-a".to_string(), resync_id: "rid-a".to_string(),
last_update: Some(datetime!(2025-02-02 00:00 UTC)), last_update: Some(datetime!(2025-02-02 00:00 UTC)),
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncCompleted, resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncCompleted,
replicated_count: 3, replicated_count: 3,
replicated_size: 9, replicated_size: 9,
bucket: "bucket-a".to_string(), bucket: "bucket-a".to_string(),
+9 -9
View File
@@ -12,10 +12,10 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::admin::storage_compat::ecstore::config::com::{STORAGE_CLASS_SUB_SYS, read_config_without_migrate}; use crate::admin::storage_compat::com::{STORAGE_CLASS_SUB_SYS, read_config_without_migrate};
use crate::admin::storage_compat::ecstore::config::set_global_storage_class; use crate::admin::storage_compat::get_global_notification_sys;
use crate::admin::storage_compat::ecstore::config::storageclass; use crate::admin::storage_compat::set_global_storage_class;
use crate::admin::storage_compat::ecstore::notification_sys::get_global_notification_sys; use crate::admin::storage_compat::storageclass;
use crate::app::context::resolve_object_store_handle; use crate::app::context::resolve_object_store_handle;
use rustfs_audit::reload_audit_config; use rustfs_audit::reload_audit_config;
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_WEBHOOK_SUB_SYS}; use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_WEBHOOK_SUB_SYS};
@@ -371,7 +371,7 @@ pub async fn signal_config_snapshot_reload() {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::admin::storage_compat::ecstore::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG}; use crate::admin::storage_compat::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG};
use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS; use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS;
use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES}; use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES};
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS}; use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
@@ -427,7 +427,7 @@ mod tests {
#[test] #[test]
fn validate_notify_subsystem_config_rejects_invalid_webhook_endpoint() { fn validate_notify_subsystem_config_rejects_invalid_webhook_endpoint() {
crate::admin::storage_compat::ecstore::config::init(); crate::admin::storage_compat::init();
let mut config = ServerConfig::new(); let mut config = ServerConfig::new();
let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults"); let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults");
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target"); let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
@@ -441,7 +441,7 @@ mod tests {
#[test] #[test]
fn validate_audit_subsystem_config_rejects_relative_queue_dir() { fn validate_audit_subsystem_config_rejects_relative_queue_dir() {
crate::admin::storage_compat::ecstore::config::init(); crate::admin::storage_compat::init();
let mut config = ServerConfig::new(); let mut config = ServerConfig::new();
let targets = config.0.get_mut(AUDIT_MQTT_SUB_SYS).expect("audit mqtt defaults"); let targets = config.0.get_mut(AUDIT_MQTT_SUB_SYS).expect("audit mqtt defaults");
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target"); let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
@@ -456,7 +456,7 @@ mod tests {
#[test] #[test]
fn validate_identity_openid_config_rejects_missing_openid_scope() { fn validate_identity_openid_config_rejects_missing_openid_scope() {
crate::admin::storage_compat::ecstore::config::init(); crate::admin::storage_compat::init();
let mut config = ServerConfig::new(); let mut config = ServerConfig::new();
let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults"); let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults");
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target"); let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
@@ -473,7 +473,7 @@ mod tests {
#[test] #[test]
fn validate_identity_openid_config_rejects_invalid_named_provider_id() { fn validate_identity_openid_config_rejects_invalid_named_provider_id() {
crate::admin::storage_compat::ecstore::config::init(); crate::admin::storage_compat::init();
let mut config = ServerConfig::new(); let mut config = ServerConfig::new();
let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults"); let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults");
let default_kvs = targets.get(DEFAULT_DELIMITER).cloned().expect("default target"); let default_kvs = targets.get(DEFAULT_DELIMITER).cloned().expect("default target");
+2 -2
View File
@@ -13,8 +13,8 @@
// limitations under the License. // limitations under the License.
use crate::admin::site_replication_identity::{deployment_id_for_endpoint, normalize_peer_map_by_identity_with}; use crate::admin::site_replication_identity::{deployment_id_for_endpoint, normalize_peer_map_by_identity_with};
use crate::admin::storage_compat::ecstore::config::com::{read_config, save_config}; use crate::admin::storage_compat::Error as StorageError;
use crate::admin::storage_compat::ecstore::error::Error as StorageError; use crate::admin::storage_compat::com::{read_config, save_config};
use crate::app::context::resolve_object_store_handle; use crate::app::context::resolve_object_store_handle;
use rustfs_madmin::PeerInfo; use rustfs_madmin::PeerInfo;
use s3s::{S3Error, S3ErrorCode, S3Result}; use s3s::{S3Error, S3ErrorCode, S3Result};
+33 -75
View File
@@ -12,78 +12,36 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
pub(crate) mod ecstore { pub(crate) use rustfs_ecstore::bucket::lifecycle::tier_last_day_stats::DailyAllTierStats;
pub(crate) mod bucket { pub(crate) use rustfs_ecstore::bucket::{
pub(crate) use rustfs_ecstore::bucket::{ bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning,
bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning, versioning_sys,
versioning_sys, };
}; pub(crate) use rustfs_ecstore::client::admin_handler_utils::AdminError;
} pub(crate) use rustfs_ecstore::config::{com, init, set_global_storage_class, storageclass};
pub(crate) use rustfs_ecstore::data_usage::load_data_usage_from_backend;
pub(crate) mod client { pub(crate) use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
pub(crate) use rustfs_ecstore::client::admin_handler_utils; #[cfg(test)]
} pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
pub(crate) mod config { #[cfg(test)]
pub(crate) use rustfs_ecstore::config::{com, init, set_global_storage_class, storageclass}; pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
} pub(crate) use rustfs_ecstore::error::{Error, StorageError};
pub(crate) use rustfs_ecstore::global::{
pub(crate) mod data_usage { GLOBAL_BOOT_TIME, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt, get_global_region,
pub(crate) use rustfs_ecstore::data_usage::load_data_usage_from_backend; global_rustfs_port,
} };
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
pub(crate) mod disk { pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
pub(crate) use rustfs_ecstore::disk::RUSTFS_META_BUCKET; pub(crate) use rustfs_ecstore::rebalance::{DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats};
#[cfg(test)] #[cfg(test)]
pub(crate) use rustfs_ecstore::disk::endpoint; pub(crate) use rustfs_ecstore::rebalance::{RebalStatus, RebalanceInfo};
} pub(crate) use rustfs_ecstore::rpc::PeerRestClient;
pub(crate) use rustfs_ecstore::store::ECStore;
pub(crate) mod endpoints { pub(crate) use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket;
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools; pub(crate) use rustfs_ecstore::tier::tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_MISSING_CREDENTIALS};
#[cfg(test)] pub(crate) use rustfs_ecstore::tier::tier_admin::TierCreds;
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints}; pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierType};
} pub(crate) use rustfs_ecstore::tier::tier_handlers::{
ERR_TIER_ALREADY_EXISTS, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND,
pub(crate) mod error { };
pub(crate) use rustfs_ecstore::error::{Error, StorageError};
}
pub(crate) mod global {
pub(crate) use rustfs_ecstore::global::{
GLOBAL_BOOT_TIME, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt, get_global_region,
global_rustfs_port,
};
}
pub(crate) mod metrics_realtime {
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
}
pub(crate) mod notification_sys {
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
}
pub(crate) mod rebalance {
pub(crate) use rustfs_ecstore::rebalance::{
DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::rebalance::{RebalStatus, RebalanceInfo};
}
pub(crate) mod rpc {
pub(crate) use rustfs_ecstore::rpc::PeerRestClient;
}
pub(crate) mod store {
pub(crate) use rustfs_ecstore::store::ECStore;
}
pub(crate) mod store_utils {
pub(crate) use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket;
}
pub(crate) mod tier {
pub(crate) use rustfs_ecstore::tier::{tier, tier_admin, tier_config, tier_handlers};
}
}
+6 -8
View File
@@ -15,13 +15,11 @@
//! Admin application use-case contracts. //! Admin application use-case contracts.
use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context}; use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
use crate::app::storage_compat::ecstore::admin_server_info::get_server_info; use crate::app::storage_compat::ECStore;
use crate::app::storage_compat::ecstore::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend}; use crate::app::storage_compat::EndpointServerPools;
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools; use crate::app::storage_compat::get_server_info;
use crate::app::storage_compat::ecstore::pools::{ use crate::app::storage_compat::{PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free};
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free, use crate::app::storage_compat::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend};
};
use crate::app::storage_compat::ecstore::store::ECStore;
use crate::capacity::resolve_admin_used_capacity; use crate::capacity::resolve_admin_used_capacity;
use crate::error::ApiError; use crate::error::ApiError;
use crate::server::{DependencyReadiness, collect_dependency_readiness as collect_runtime_dependency_readiness}; use crate::server::{DependencyReadiness, collect_dependency_readiness as collect_runtime_dependency_readiness};
@@ -318,7 +316,7 @@ impl DefaultAdminUsecase {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::app::storage_compat::ecstore::pools::{PoolDecommissionInfo, PoolStatus}; use crate::app::storage_compat::{PoolDecommissionInfo, PoolStatus};
use time::OffsetDateTime; use time::OffsetDateTime;
#[tokio::test] #[tokio::test]
+7 -9
View File
@@ -20,7 +20,11 @@ use crate::admin::handlers::site_replication::{
use crate::app::context::{ use crate::app::context::{
AppContext, default_notify_interface, get_global_app_context, resolve_object_store_handle_for_context, AppContext, default_notify_interface, get_global_app_context, resolve_object_store_handle_for_context,
}; };
use crate::app::storage_compat::ecstore::bucket::{ use crate::app::storage_compat::ECStore;
use crate::app::storage_compat::StorageError;
use crate::app::storage_compat::get_global_notification_sys;
use crate::app::storage_compat::object_api_utils::to_s3s_etag;
use crate::app::storage_compat::{
bucket_target_sys::BucketTargetSys, bucket_target_sys::BucketTargetSys,
lifecycle::bucket_lifecycle_ops::{ lifecycle::bucket_lifecycle_ops::{
enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, validate_transition_tier, enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, validate_transition_tier,
@@ -38,10 +42,6 @@ use crate::app::storage_compat::ecstore::bucket::{
versioning::VersioningApi, versioning::VersioningApi,
versioning_sys::BucketVersioningSys, versioning_sys::BucketVersioningSys,
}; };
use crate::app::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
use crate::app::storage_compat::ecstore::error::StorageError;
use crate::app::storage_compat::ecstore::notification_sys::get_global_notification_sys;
use crate::app::storage_compat::ecstore::store::ECStore;
use crate::auth::get_condition_values_with_client_info; use crate::auth::get_condition_values_with_client_info;
use crate::error::ApiError; use crate::error::ApiError;
use crate::server::RemoteAddr; use crate::server::RemoteAddr;
@@ -1628,9 +1628,7 @@ impl DefaultBucketUsecase {
} }
}; };
if let Err(err) = if let Err(err) = crate::app::storage_compat::lifecycle::lifecycle::Lifecycle::validate(&input_cfg, &rcfg).await {
crate::app::storage_compat::ecstore::bucket::lifecycle::lifecycle::Lifecycle::validate(&input_cfg, &rcfg).await
{
return Err(s3_error!(InvalidArgument, "{err}")); return Err(s3_error!(InvalidArgument, "{err}"));
} }
@@ -2280,7 +2278,7 @@ mod tests {
BucketTargets { BucketTargets {
targets: arns targets: arns
.iter() .iter()
.map(|arn| crate::app::storage_compat::ecstore::bucket::target::BucketTarget { .map(|arn| crate::app::storage_compat::target::BucketTarget {
arn: (*arn).to_string(), arn: (*arn).to_string(),
target_type: BucketTargetType::ReplicationService, target_type: BucketTargetType::ReplicationService,
..Default::default() ..Default::default()
+2 -7
View File
@@ -12,12 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::app::storage_compat::ecstore::{ use crate::app::storage_compat::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, metadata_sys};
bucket::metadata_sys,
disk::endpoint::Endpoint,
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
store::ECStore,
};
use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager}; use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager};
use rustfs_storage_api::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _}; use rustfs_storage_api::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _};
@@ -82,7 +77,7 @@ async fn setup_capacity_dirty_scope_env() -> (Vec<PathBuf>, Arc<ECStore>) {
}; };
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
crate::app::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone()) crate::app::storage_compat::init_local_disks(endpoint_pools.clone())
.await .await
.unwrap(); .unwrap();
+9 -9
View File
@@ -17,11 +17,11 @@ use super::handles::{
default_bucket_metadata_interface, default_endpoints_interface, default_kms_runtime_interface, default_bucket_metadata_interface, default_endpoints_interface, default_kms_runtime_interface,
default_server_config_interface, default_tier_config_interface, default_server_config_interface, default_tier_config_interface,
}; };
use crate::app::storage_compat::ecstore::bucket::metadata_sys::BucketMetadataSys; use crate::app::storage_compat::ECStore;
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools; use crate::app::storage_compat::EndpointServerPools;
use crate::app::storage_compat::ecstore::new_object_layer_fn; use crate::app::storage_compat::TierConfigMgr;
use crate::app::storage_compat::ecstore::store::ECStore; use crate::app::storage_compat::metadata_sys::BucketMetadataSys;
use crate::app::storage_compat::ecstore::tier::tier::TierConfigMgr; use crate::app::storage_compat::new_object_layer_fn;
#[cfg(test)] #[cfg(test)]
use crate::config::RustFSBufferConfig; use crate::config::RustFSBufferConfig;
use rustfs_config::server_config::Config; use rustfs_config::server_config::Config;
@@ -126,10 +126,10 @@ mod tests {
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface, BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
ServerConfigInterface, TierConfigInterface, ServerConfigInterface, TierConfigInterface,
}; };
use crate::app::storage_compat::ecstore::disk::endpoint::Endpoint; use crate::app::storage_compat::Endpoint;
use crate::app::storage_compat::ecstore::endpoints::{Endpoints, PoolEndpoints}; use crate::app::storage_compat::init_local_disks;
use crate::app::storage_compat::ecstore::new_object_layer_fn; use crate::app::storage_compat::new_object_layer_fn;
use crate::app::storage_compat::ecstore::store::init_local_disks; use crate::app::storage_compat::{Endpoints, PoolEndpoints};
use crate::config::{RustFSBufferConfig, WorkloadProfile}; use crate::config::{RustFSBufferConfig, WorkloadProfile};
use rustfs_iam::{store::object::ObjectStore, sys::IamSys}; use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use std::path::PathBuf; use std::path::PathBuf;
+1 -1
View File
@@ -21,7 +21,7 @@ use super::interfaces::{
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface, BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface, NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface,
}; };
use crate::app::storage_compat::ecstore::{set_object_store_resolver, store::ECStore}; use crate::app::storage_compat::{ECStore, set_object_store_resolver};
use rustfs_iam::{store::object::ObjectStore, sys::IamSys}; use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager; use rustfs_kms::KmsServiceManager;
use std::sync::{Arc, OnceLock}; use std::sync::{Arc, OnceLock};
+4 -4
View File
@@ -16,10 +16,10 @@ use super::interfaces::{
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface, BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface, NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface,
}; };
use crate::app::storage_compat::ecstore::bucket::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys}; use crate::app::storage_compat::EndpointServerPools;
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools; use crate::app::storage_compat::TierConfigMgr;
use crate::app::storage_compat::ecstore::global::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr}; use crate::app::storage_compat::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys};
use crate::app::storage_compat::ecstore::tier::tier::TierConfigMgr; use crate::app::storage_compat::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr};
use crate::config::{RustFSBufferConfig, get_global_buffer_config}; use crate::config::{RustFSBufferConfig, get_global_buffer_config};
use async_trait::async_trait; use async_trait::async_trait;
use rustfs_config::server_config::Config; use rustfs_config::server_config::Config;
+3 -3
View File
@@ -12,9 +12,9 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::app::storage_compat::ecstore::bucket::metadata_sys::BucketMetadataSys; use crate::app::storage_compat::EndpointServerPools;
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools; use crate::app::storage_compat::TierConfigMgr;
use crate::app::storage_compat::ecstore::tier::tier::TierConfigMgr; use crate::app::storage_compat::metadata_sys::BucketMetadataSys;
use crate::config::RustFSBufferConfig; use crate::config::RustFSBufferConfig;
use async_trait::async_trait; use async_trait::async_trait;
use rustfs_config::server_config::Config; use rustfs_config::server_config::Config;
+11 -17
View File
@@ -14,19 +14,13 @@
use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase}; use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase};
use crate::app::bucket_usecase::DefaultBucketUsecase; use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::storage_compat::ecstore::{ use crate::app::storage_compat::{
bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG}, ECStore, Endpoint, EndpointServerPools, Endpoints, GLOBAL_TierConfigMgr, PoolEndpoints, TierConfig, TierType, WarmBackend,
bucket::metadata_sys, WarmBackendGetOpts,
client::object_api_utils::to_s3s_etag, metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG},
client::transition_api::{ReadCloser, ReaderImpl}, metadata_sys,
disk::endpoint::Endpoint, object_api_utils::to_s3s_etag,
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}, transition_api::{ReadCloser, ReaderImpl},
global::GLOBAL_TierConfigMgr,
store::ECStore,
tier::{
tier_config::{TierConfig, TierType},
warm_backend::{WarmBackend, WarmBackendGetOpts},
},
}; };
use crate::storage::ecfs::FS; use crate::storage::ecfs::FS;
use crate::storage::{ use crate::storage::{
@@ -113,7 +107,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]); let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
crate::app::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone()) crate::app::storage_compat::init_local_disks(endpoint_pools.clone())
.await .await
.unwrap(); .unwrap();
@@ -132,7 +126,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
let buckets = buckets_list.into_iter().map(|v| v.name).collect(); let buckets = buckets_list.into_iter().map(|v| v.name).collect();
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await; metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await; crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone())); let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
@@ -932,7 +926,7 @@ async fn delete_transitioned_object_removes_remote_tier_copy_via_usecase() {
.expect("Failed to set lifecycle configuration"); .expect("Failed to set lifecycle configuration");
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await; let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
ecstore.clone(), ecstore.clone(),
bucket.as_str(), bucket.as_str(),
) )
@@ -992,7 +986,7 @@ async fn lifecycle_transition_marks_dirty_disks_for_capacity_manager() {
.expect("Failed to set lifecycle configuration"); .expect("Failed to set lifecycle configuration");
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await; let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
ecstore.clone(), ecstore.clone(),
bucket.as_str(), bucket.as_str(),
) )
+36 -56
View File
@@ -16,22 +16,22 @@
use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context}; use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write}; use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write};
use crate::app::storage_compat::ecstore::bucket::quota::checker::QuotaChecker; use crate::app::storage_compat::ECStore;
use crate::app::storage_compat::ecstore::bucket::{ use crate::app::storage_compat::is_disk_compressible;
use crate::app::storage_compat::is_valid_storage_class;
use crate::app::storage_compat::object_api_utils::to_s3s_etag;
use crate::app::storage_compat::quota::checker::QuotaChecker;
#[cfg(test)]
use crate::app::storage_compat::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
use crate::app::storage_compat::{HashReader, WritePlan};
use crate::app::storage_compat::{StorageError, is_err_object_not_found, is_err_version_not_found};
use crate::app::storage_compat::{
lifecycle::{bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::enqueue_transition_immediate}, lifecycle::{bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::enqueue_transition_immediate},
metadata_sys, metadata_sys,
quota::QuotaOperation, quota::QuotaOperation,
replication::{get_must_replicate_options, must_replicate, schedule_replication}, replication::{get_must_replicate_options, must_replicate, schedule_replication},
versioning_sys::BucketVersioningSys, versioning_sys::BucketVersioningSys,
}; };
use crate::app::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
use crate::app::storage_compat::ecstore::compress::is_disk_compressible;
use crate::app::storage_compat::ecstore::error::{StorageError, is_err_object_not_found, is_err_version_not_found};
#[cfg(test)]
use crate::app::storage_compat::ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
use crate::app::storage_compat::ecstore::rio::{HashReader, WritePlan};
use crate::app::storage_compat::ecstore::set_disk::is_valid_storage_class;
use crate::app::storage_compat::ecstore::store::ECStore;
use crate::capacity::record_capacity_write; use crate::capacity::record_capacity_write;
use crate::error::ApiError; use crate::error::ApiError;
use crate::storage::access::has_bypass_governance_header; use crate::storage::access::has_bypass_governance_header;
@@ -479,7 +479,7 @@ impl DefaultMultipartUsecase {
)); ));
} }
// Update quota tracking after successful multipart upload // Update quota tracking after successful multipart upload
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_write_memory( crate::app::storage_compat::record_bucket_object_write_memory(
&bucket, &bucket,
previous_current_size, previous_current_size,
obj_info.size.max(0) as u64, obj_info.size.max(0) as u64,
@@ -672,7 +672,7 @@ impl DefaultMultipartUsecase {
rustfs_utils::http::insert_str( rustfs_utils::http::insert_str(
&mut metadata, &mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION, rustfs_utils::http::SUFFIX_COMPRESSION,
crate::app::storage_compat::ecstore::rio::compression_metadata_value(CompressionAlgorithm::default()), crate::app::storage_compat::compression_metadata_value(CompressionAlgorithm::default()),
); );
} }
@@ -891,18 +891,13 @@ impl DefaultMultipartUsecase {
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?; .ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
let ssec_write = match ssec_material.key_kind { let ssec_write = match ssec_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => { crate::storage::sse::EncryptionKeyKind::Object => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key( crate::app::storage_compat::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
ssec_material.key_bytes,
part_id as u32,
)
}
crate::storage::sse::EncryptionKeyKind::Direct => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
ssec_material.key_bytes,
ssec_material.base_nonce,
part_id,
)
} }
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
ssec_material.key_bytes,
ssec_material.base_nonce,
part_id,
),
}; };
write_plan = write_plan.with_encryption(ssec_write); write_plan = write_plan.with_encryption(ssec_write);
(Some(ssec_material.server_side_encryption), ssec_material.kms_key_id) (Some(ssec_material.server_side_encryption), ssec_material.kms_key_id)
@@ -918,18 +913,13 @@ impl DefaultMultipartUsecase {
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?; .ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
let managed_write = match managed_material.key_kind { let managed_write = match managed_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => { crate::storage::sse::EncryptionKeyKind::Object => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key( crate::app::storage_compat::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
managed_material.key_bytes,
part_id as u32,
)
}
crate::storage::sse::EncryptionKeyKind::Direct => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
managed_material.key_bytes,
managed_material.base_nonce,
part_id,
)
} }
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
managed_material.key_bytes,
managed_material.base_nonce,
part_id,
),
}; };
write_plan = write_plan.with_encryption(managed_write); write_plan = write_plan.with_encryption(managed_write);
(Some(server_side_encryption), ssekms_key_id) (Some(server_side_encryption), ssekms_key_id)
@@ -1244,18 +1234,13 @@ impl DefaultMultipartUsecase {
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?; .ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
let ssec_write = match ssec_material.key_kind { let ssec_write = match ssec_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => { crate::storage::sse::EncryptionKeyKind::Object => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key( crate::app::storage_compat::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
ssec_material.key_bytes,
part_id as u32,
)
}
crate::storage::sse::EncryptionKeyKind::Direct => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
ssec_material.key_bytes,
ssec_material.base_nonce,
part_id,
)
} }
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
ssec_material.key_bytes,
ssec_material.base_nonce,
part_id,
),
}; };
write_plan = write_plan.with_encryption(ssec_write); write_plan = write_plan.with_encryption(ssec_write);
( (
@@ -1275,18 +1260,13 @@ impl DefaultMultipartUsecase {
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?; .ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
let managed_write = match managed_material.key_kind { let managed_write = match managed_material.key_kind {
crate::storage::sse::EncryptionKeyKind::Object => { crate::storage::sse::EncryptionKeyKind::Object => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key( crate::app::storage_compat::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
managed_material.key_bytes,
part_id as u32,
)
}
crate::storage::sse::EncryptionKeyKind::Direct => {
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
managed_material.key_bytes,
managed_material.base_nonce,
part_id,
)
} }
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
managed_material.key_bytes,
managed_material.base_nonce,
part_id,
),
}; };
write_plan = write_plan.with_encryption(managed_write); write_plan = write_plan.with_encryption(managed_write);
(Some(server_side_encryption), ssekms_key_id, mp_info.user_defined.clone()) (Some(server_side_encryption), ssekms_key_id, mp_info.user_defined.clone())
+26 -31
View File
@@ -48,8 +48,18 @@ use metrics::{counter, histogram};
use pin_project_lite::pin_project; use pin_project_lite::pin_project;
use rustfs_object_capacity::capacity_manager::get_capacity_manager; use rustfs_object_capacity::capacity_manager::get_capacity_manager;
// Performance metrics recording (with zero-copy-metrics integration) // Performance metrics recording (with zero-copy-metrics integration)
use crate::app::storage_compat::ecstore::bucket::quota::checker::QuotaChecker; use crate::app::storage_compat::ECStore;
use crate::app::storage_compat::ecstore::bucket::{ use crate::app::storage_compat::object_api_utils::to_s3s_etag;
use crate::app::storage_compat::quota::checker::QuotaChecker;
use crate::app::storage_compat::storageclass;
use crate::app::storage_compat::{DiskError, is_all_buckets_not_found};
use crate::app::storage_compat::{DynReader, HashReader, WritePlan, wrap_reader};
use crate::app::storage_compat::{
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
use crate::app::storage_compat::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
use crate::app::storage_compat::{get_lock_acquire_timeout, is_valid_storage_class};
use crate::app::storage_compat::{
lifecycle::{ lifecycle::{
bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_audit::LcEventSrc,
bucket_lifecycle_ops::{RestoreRequestOps, enqueue_transition_immediate, post_restore_opts}, bucket_lifecycle_ops::{RestoreRequestOps, enqueue_transition_immediate, post_restore_opts},
@@ -69,16 +79,6 @@ use crate::app::storage_compat::ecstore::bucket::{
versioning::VersioningApi, versioning::VersioningApi,
versioning_sys::BucketVersioningSys, versioning_sys::BucketVersioningSys,
}; };
use crate::app::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
use crate::app::storage_compat::ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
use crate::app::storage_compat::ecstore::config::storageclass;
use crate::app::storage_compat::ecstore::disk::{error::DiskError, error_reduce::is_all_buckets_not_found};
use crate::app::storage_compat::ecstore::error::{
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
use crate::app::storage_compat::ecstore::rio::{DynReader, HashReader, WritePlan, wrap_reader};
use crate::app::storage_compat::ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
use crate::app::storage_compat::ecstore::store::ECStore;
use rustfs_concurrency::GetObjectQueueSnapshot; use rustfs_concurrency::GetObjectQueueSnapshot;
use rustfs_filemeta::{ use rustfs_filemeta::{
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType, REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType,
@@ -276,12 +276,12 @@ async fn enqueue_transitioned_delete_cleanup(
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup); let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup);
let je = if opts.delete_prefix { let je = if opts.delete_prefix {
crate::app::storage_compat::ecstore::bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry( crate::app::storage_compat::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(
&existing.transitioned_object, &existing.transitioned_object,
) )
} else { } else {
let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok()); let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok());
crate::app::storage_compat::ecstore::bucket::lifecycle::tier_sweeper::transitioned_delete_journal_entry( crate::app::storage_compat::lifecycle::tier_sweeper::transitioned_delete_journal_entry(
version_id, version_id,
opts.versioned, opts.versioned,
opts.version_suspended, opts.version_suspended,
@@ -292,10 +292,9 @@ async fn enqueue_transitioned_delete_cleanup(
return Ok(()); return Ok(());
}; };
crate::app::storage_compat::ecstore::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(store, &je) crate::app::storage_compat::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(store, &je).await?;
.await?;
let mut expiry_state = crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState let mut expiry_state = crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
.write() .write()
.await; .await;
if let Err(err) = expiry_state.enqueue_tier_journal_entry(&je).await { if let Err(err) = expiry_state.enqueue_tier_journal_entry(&je).await {
@@ -1540,7 +1539,7 @@ impl DefaultObjectUsecase {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
async fn prepare_get_object_read( async fn prepare_get_object_read(
req: &S3Request<GetObjectInput>, req: &S3Request<GetObjectInput>,
store: &crate::app::storage_compat::ecstore::store::ECStore, store: &crate::app::storage_compat::ECStore,
manager: &ConcurrencyManager, manager: &ConcurrencyManager,
bucket: &str, bucket: &str,
key: &str, key: &str,
@@ -2134,7 +2133,7 @@ impl DefaultObjectUsecase {
insert_str( insert_str(
&mut metadata, &mut metadata,
SUFFIX_COMPRESSION, SUFFIX_COMPRESSION,
crate::app::storage_compat::ecstore::rio::compression_metadata_value(algorithm), crate::app::storage_compat::compression_metadata_value(algorithm),
); );
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
@@ -2149,7 +2148,7 @@ impl DefaultObjectUsecase {
insert_str( insert_str(
&mut opts.user_defined, &mut opts.user_defined,
SUFFIX_COMPRESSION, SUFFIX_COMPRESSION,
crate::app::storage_compat::ecstore::rio::compression_metadata_value(algorithm), crate::app::storage_compat::compression_metadata_value(algorithm),
); );
insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string()); insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string());
@@ -2341,7 +2340,7 @@ impl DefaultObjectUsecase {
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await; maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
// Fast in-memory update for immediate quota and admin usage consistency // Fast in-memory update for immediate quota and admin usage consistency
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_write_memory( crate::app::storage_compat::record_bucket_object_write_memory(
&bucket, &bucket,
previous_current_size, previous_current_size,
obj_info.size.max(0) as u64, obj_info.size.max(0) as u64,
@@ -3211,7 +3210,7 @@ impl DefaultObjectUsecase {
insert_str( insert_str(
&mut compress_metadata, &mut compress_metadata,
SUFFIX_COMPRESSION, SUFFIX_COMPRESSION,
crate::app::storage_compat::ecstore::rio::compression_metadata_value(CompressionAlgorithm::default()), crate::app::storage_compat::compression_metadata_value(CompressionAlgorithm::default()),
); );
insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string()); insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
} else { } else {
@@ -3304,12 +3303,8 @@ impl DefaultObjectUsecase {
// Update quota tracking after successful copy // Update quota tracking after successful copy
if has_bucket_metadata { if has_bucket_metadata {
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_write_memory( crate::app::storage_compat::record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64)
&bucket, .await;
previous_current_size,
oi.size.max(0) as u64,
)
.await;
} }
let raw_dest_version = oi.version_id.map(|v| v.to_string()); let raw_dest_version = oi.version_id.map(|v| v.to_string());
@@ -3586,7 +3581,7 @@ impl DefaultObjectUsecase {
); );
} }
let size = object_sizes[i].max(0) as u64; let size = object_sizes[i].max(0) as u64;
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_delete_memory( crate::app::storage_compat::record_bucket_object_delete_memory(
&bucket, &bucket,
size, size,
existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(), existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
@@ -3824,7 +3819,7 @@ impl DefaultObjectUsecase {
} }
// Fast in-memory update for immediate quota and admin usage consistency // Fast in-memory update for immediate quota and admin usage consistency
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_delete_memory( crate::app::storage_compat::record_bucket_object_delete_memory(
&bucket, &bucket,
obj_info.size.max(0) as u64, obj_info.size.max(0) as u64,
opts.version_id.is_none(), opts.version_id.is_none(),
@@ -4757,7 +4752,7 @@ impl DefaultObjectUsecase {
insert_str( insert_str(
&mut metadata, &mut metadata,
SUFFIX_COMPRESSION, SUFFIX_COMPRESSION,
crate::app::storage_compat::ecstore::rio::compression_metadata_value(algorithm), crate::app::storage_compat::compression_metadata_value(algorithm),
); );
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string()); insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
+47 -93
View File
@@ -12,96 +12,50 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
pub(crate) mod ecstore { pub(crate) use rustfs_ecstore::admin_server_info::get_server_info;
pub(crate) use rustfs_ecstore::global::{new_object_layer_fn, set_object_store_resolver}; pub(crate) use rustfs_ecstore::bucket::{
bucket_target_sys, lifecycle, metadata, metadata_sys, object_lock, policy_sys, quota, replication, tagging, target, utils,
pub(crate) mod admin_server_info { versioning, versioning_sys,
pub(crate) use rustfs_ecstore::admin_server_info::get_server_info; };
} pub(crate) use rustfs_ecstore::client::object_api_utils;
#[cfg(test)]
pub(crate) mod bucket { pub(crate) use rustfs_ecstore::client::transition_api;
pub(crate) use rustfs_ecstore::bucket::{ pub(crate) use rustfs_ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
bucket_target_sys, lifecycle, metadata, metadata_sys, object_lock, policy_sys, quota, replication, tagging, target, pub(crate) use rustfs_ecstore::config::storageclass;
utils, versioning, versioning_sys, pub(crate) use rustfs_ecstore::data_usage::{
}; apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory,
} record_bucket_object_write_memory,
};
pub(crate) mod client { #[cfg(test)]
pub(crate) use rustfs_ecstore::client::object_api_utils; pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
#[cfg(test)] pub(crate) use rustfs_ecstore::disk::error::DiskError;
pub(crate) use rustfs_ecstore::client::transition_api; pub(crate) use rustfs_ecstore::disk::error_reduce::is_all_buckets_not_found;
} pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
#[cfg(test)]
pub(crate) mod compress { pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
pub(crate) use rustfs_ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; pub(crate) use rustfs_ecstore::error::{
} Error, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
pub(crate) mod config { #[cfg(test)]
pub(crate) use rustfs_ecstore::config::storageclass; pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
} pub(crate) use rustfs_ecstore::global::{
get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr, new_object_layer_fn, set_object_store_resolver,
pub(crate) mod data_usage { };
pub(crate) use rustfs_ecstore::data_usage::{ pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory, pub(crate) use rustfs_ecstore::pools::{
record_bucket_object_write_memory, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
}; };
} #[cfg(test)]
pub(crate) use rustfs_ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader};
pub(crate) mod disk { pub(crate) use rustfs_ecstore::rio::{
#[cfg(test)] DynReader, HashReader, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader,
pub(crate) use rustfs_ecstore::disk::endpoint; };
pub(crate) use rustfs_ecstore::disk::{error, error_reduce}; pub(crate) use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
} pub(crate) use rustfs_ecstore::store::ECStore;
#[cfg(test)]
pub(crate) mod endpoints { pub(crate) use rustfs_ecstore::store::init_local_disks;
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools; pub(crate) use rustfs_ecstore::tier::tier::TierConfigMgr;
#[cfg(test)] #[cfg(test)]
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints}; pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierType};
} #[cfg(test)]
pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, WarmBackendGetOpts};
pub(crate) mod error {
pub(crate) use rustfs_ecstore::error::{
Error, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
}
pub(crate) mod global {
#[cfg(test)]
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
pub(crate) use rustfs_ecstore::global::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr};
}
pub(crate) mod notification_sys {
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
}
pub(crate) mod pools {
pub(crate) use rustfs_ecstore::pools::{
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
};
}
pub(crate) mod rio {
#[cfg(test)]
pub(crate) use rustfs_ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader};
pub(crate) use rustfs_ecstore::rio::{
DynReader, HashReader, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader,
};
}
pub(crate) mod set_disk {
pub(crate) use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
}
pub(crate) mod store {
pub(crate) use rustfs_ecstore::store::ECStore;
#[cfg(test)]
pub(crate) use rustfs_ecstore::store::init_local_disks;
}
pub(crate) mod tier {
pub(crate) use rustfs_ecstore::tier::tier;
#[cfg(test)]
pub(crate) use rustfs_ecstore::tier::{tier_config, warm_backend};
}
}
+2 -2
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::storage_compat::ecstore::disk::DiskAPI; use crate::storage_compat::DiskAPI;
use rustfs_io_metrics::capacity_metrics::{ use rustfs_io_metrics::capacity_metrics::{
record_capacity_cache_hit, record_capacity_cache_miss, record_capacity_cache_served, record_capacity_refresh_request, record_capacity_cache_hit, record_capacity_cache_miss, record_capacity_cache_served, record_capacity_refresh_request,
record_capacity_scan_mode, record_capacity_scan_mode,
@@ -263,7 +263,7 @@ pub async fn init_capacity_management_for_local_disks() {
"Capacity manager state changed" "Capacity manager state changed"
); );
let disks = crate::storage_compat::ecstore::store::all_local_disk().await; let disks = crate::storage_compat::all_local_disk().await;
if disks.is_empty() { if disks.is_empty() {
warn!( warn!(
component = LOG_COMPONENT_CAPACITY, component = LOG_COMPONENT_CAPACITY,
+2 -2
View File
@@ -16,7 +16,7 @@
#[allow(unsafe_op_in_unsafe_fn)] #[allow(unsafe_op_in_unsafe_fn)]
mod tests { mod tests {
use crate::config::{CommandResult, Config, Opt, TlsCommands}; use crate::config::{CommandResult, Config, Opt, TlsCommands};
use crate::storage_compat::ecstore::disks_layout::DisksLayout; use crate::storage_compat::DisksLayout;
use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_OBS_ENDPOINT, RUSTFS_REGION}; use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_OBS_ENDPOINT, RUSTFS_REGION};
use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY}; use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY};
use serial_test::serial; use serial_test::serial;
@@ -262,7 +262,7 @@ mod tests {
#[test] #[test]
#[serial] #[serial]
fn test_volumes_and_disk_layout_parsing() { fn test_volumes_and_disk_layout_parsing() {
use crate::storage_compat::ecstore::disks_layout::DisksLayout; use crate::storage_compat::DisksLayout;
// Test case 1: Single volume path // Test case 1: Single volume path
let args = vec!["rustfs", "/data/vol1"]; let args = vec!["rustfs", "/data/vol1"];
+9 -18
View File
@@ -51,20 +51,11 @@ use crate::init::{add_bucket_notification_configuration, init_buffer_profile_sys
use crate::server::{ShutdownHandle, shutdown_event_notifier, start_http_server, stop_audit_system}; use crate::server::{ShutdownHandle, shutdown_event_notifier, start_http_server, stop_audit_system};
use crate::startup_fs_guard::enforce_unsupported_fs_policy; use crate::startup_fs_guard::enforce_unsupported_fs_policy;
use crate::startup_iam::{bootstrap_or_defer_iam_init, publish_ready_for_iam_bootstrap}; use crate::startup_iam::{bootstrap_or_defer_iam_init, publish_ready_for_iam_bootstrap};
use crate::storage_compat::ecstore::store::init_lock_clients; use crate::storage_compat::init_lock_clients;
use crate::storage_compat::ecstore::{ use crate::storage_compat::{
bucket::replication::init_background_replication, ECStore, EndpointServerPools, init as init_ecstore_config, init_background_replication, init_bucket_metadata_sys,
bucket::{ init_global_config_sys, init_local_disks, set_global_endpoints, set_global_rustfs_port, try_migrate_bucket_metadata,
metadata_sys::init_bucket_metadata_sys, try_migrate_iam_config, try_migrate_server_config, update_erasure_type,
migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
},
config as ecconfig,
endpoints::EndpointServerPools,
global::set_global_rustfs_port,
set_global_endpoints,
store::ECStore,
store::init_local_disks,
update_erasure_type,
}; };
use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr}; use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
use rustfs_credentials::init_global_action_credentials; use rustfs_credentials::init_global_action_credentials;
@@ -333,7 +324,7 @@ impl RustFSServerBuilder {
let region = region_str let region = region_str
.parse() .parse()
.map_err(|e| ServerError::Init(format!("invalid region '{region_str}': {e}")))?; .map_err(|e| ServerError::Init(format!("invalid region '{region_str}': {e}")))?;
crate::storage_compat::ecstore::global::set_global_region(region); crate::storage_compat::set_global_region(region);
} }
let server_port = server_addr.port(); let server_port = server_addr.port();
@@ -389,12 +380,12 @@ impl RustFSServerBuilder {
} }
}; };
ecconfig::init(); init_ecstore_config();
ecconfig::try_migrate_server_config(store.clone()).await; try_migrate_server_config(store.clone()).await;
// Global config system (with retry). // Global config system (with retry).
let mut retry = 0; let mut retry = 0;
while let Err(e) = ecconfig::init_global_config_sys(store.clone()).await { while let Err(e) = init_global_config_sys(store.clone()).await {
retry += 1; retry += 1;
if retry > 15 { if retry > 15 {
shutdown_embedded_server(); shutdown_embedded_server();
+2 -2
View File
@@ -12,8 +12,8 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::storage_compat::ecstore::bucket::quota::QuotaError; use crate::storage_compat::StorageError;
use crate::storage_compat::ecstore::error::StorageError; use crate::storage_compat::quota::QuotaError;
use rustfs_storage_api::HTTPRangeError; use rustfs_storage_api::HTTPRangeError;
use s3s::{S3Error, S3ErrorCode}; use s3s::{S3Error, S3ErrorCode};
+2 -2
View File
@@ -14,7 +14,7 @@
use crate::server::ShutdownHandle; use crate::server::ShutdownHandle;
use crate::storage::{process_lambda_configurations, process_queue_configurations, process_topic_configurations}; use crate::storage::{process_lambda_configurations, process_queue_configurations, process_topic_configurations};
use crate::storage_compat::ecstore::bucket::metadata_sys; use crate::storage_compat::metadata_sys;
use crate::{admin, config, version}; use crate::{admin, config, version};
use rustfs_config::{ use rustfs_config::{
DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK, DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK,
@@ -157,7 +157,7 @@ fn arn_to_target_id(arn_str: &str) -> Result<rustfs_targets::arn::TargetID, Targ
/// * `buckets` - A vector of bucket names to process /// * `buckets` - A vector of bucket names to process
#[instrument(skip_all)] #[instrument(skip_all)]
pub async fn add_bucket_notification_configuration(buckets: Vec<String>) { pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
let global_region = crate::storage_compat::ecstore::global::get_global_region(); let global_region = crate::storage_compat::get_global_region();
let region = global_region let region = global_region
.as_ref() .as_ref()
.filter(|r| !r.as_str().is_empty()) .filter(|r| !r.as_str().is_empty())
+1 -1
View File
@@ -14,7 +14,7 @@
use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store}; use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store};
use crate::app::context::resolve_server_config; use crate::app::context::resolve_server_config;
use crate::storage_compat::ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook}; use crate::storage_compat::{EcstoreEventArgs, register_event_dispatch_hook};
use rustfs_notify::EventArgs as NotifyEventArgs; use rustfs_notify::EventArgs as NotifyEventArgs;
use rustfs_s3_types::EventName; use rustfs_s3_types::EventName;
use std::net::SocketAddr; use std::net::SocketAddr;
+1 -1
View File
@@ -33,7 +33,7 @@ use crate::server::{
use crate::storage; use crate::storage;
use crate::storage::rpc::InternodeRpcService; use crate::storage::rpc::InternodeRpcService;
use crate::storage::tonic_service::make_server; use crate::storage::tonic_service::make_server;
use crate::storage_compat::ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature}; use crate::storage_compat::{TONIC_RPC_PREFIX, verify_rpc_signature};
use bytes::Bytes; use bytes::Bytes;
use http::{HeaderMap, Method, Request as HttpRequest, Response}; use http::{HeaderMap, Method, Request as HttpRequest, Response};
use hyper_util::{ use hyper_util::{
+3 -3
View File
@@ -12,9 +12,9 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::storage_compat::ecstore::{ use crate::storage_compat::{
config::com::{read_config, save_config}, EcstoreError as StorageError,
error::Error as StorageError, com::{read_config, save_config},
resolve_object_store_handle, resolve_object_store_handle,
}; };
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
+10 -13
View File
@@ -14,9 +14,9 @@
use crate::server::{ServiceState, ServiceStateManager}; use crate::server::{ServiceState, ServiceStateManager};
use crate::server::{has_path_prefix, is_table_catalog_path}; use crate::server::{has_path_prefix, is_table_catalog_path};
use crate::storage_compat::ecstore::global::is_dist_erasure; use crate::storage_compat::is_dist_erasure;
use crate::storage_compat::ecstore::global::{get_global_endpoints_opt, get_global_lock_clients}; use crate::storage_compat::resolve_object_store_handle;
use crate::storage_compat::ecstore::resolve_object_store_handle; use crate::storage_compat::{get_global_endpoints_opt, get_global_lock_clients};
use bytes::Bytes; use bytes::Bytes;
use http::{Request as HttpRequest, Response, StatusCode}; use http::{Request as HttpRequest, Response, StatusCode};
use http_body::Body; use http_body::Body;
@@ -496,13 +496,10 @@ async fn collect_storage_readiness_uncached() -> bool {
} }
} }
fn set_lock_quorum_status( fn set_lock_quorum_status(online_hosts: &HashSet<String>, set_endpoints: &[crate::storage_compat::Endpoint]) -> LockQuorumStatus {
online_hosts: &HashSet<String>,
set_endpoints: &[crate::storage_compat::ecstore::disk::endpoint::Endpoint],
) -> LockQuorumStatus {
let total_clients = set_endpoints let total_clients = set_endpoints
.iter() .iter()
.map(crate::storage_compat::ecstore::disk::endpoint::Endpoint::host_port) .map(crate::storage_compat::Endpoint::host_port)
.filter(|host| !host.is_empty()) .filter(|host| !host.is_empty())
.collect::<HashSet<_>>(); .collect::<HashSet<_>>();
let total_clients_len = total_clients.len(); let total_clients_len = total_clients.len();
@@ -526,7 +523,7 @@ fn set_lock_quorum_status(
} }
fn aggregate_lock_quorum_status( fn aggregate_lock_quorum_status(
pool_endpoints: &crate::storage_compat::ecstore::endpoints::EndpointServerPools, pool_endpoints: &crate::storage_compat::EndpointServerPools,
online_hosts: &HashSet<String>, online_hosts: &HashSet<String>,
) -> LockQuorumStatus { ) -> LockQuorumStatus {
let mut connected_clients = 0usize; let mut connected_clients = 0usize;
@@ -842,8 +839,8 @@ mod tests {
#[test] #[test]
fn aggregate_lock_quorum_status_requires_each_set_to_meet_quorum() { fn aggregate_lock_quorum_status_requires_each_set_to_meet_quorum() {
use crate::storage_compat::ecstore::disk::endpoint::Endpoint; use crate::storage_compat::Endpoint;
use crate::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints};
let endpoints = vec![ let endpoints = vec![
Endpoint { Endpoint {
@@ -900,8 +897,8 @@ mod tests {
#[test] #[test]
fn aggregate_lock_quorum_status_fails_when_any_set_loses_quorum() { fn aggregate_lock_quorum_status_fails_when_any_set_loses_quorum() {
use crate::storage_compat::ecstore::disk::endpoint::Endpoint; use crate::storage_compat::Endpoint;
use crate::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints};
let endpoints = vec![ let endpoints = vec![
Endpoint { Endpoint {
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::storage_compat::ecstore::endpoints::EndpointServerPools; use crate::storage_compat::EndpointServerPools;
use rustfs_config::{ use rustfs_config::{
DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, ENV_RUSTFS_UNSUPPORTED_FS_POLICY, RUSTFS_UNSUPPORTED_FS_POLICY_FAIL, DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, ENV_RUSTFS_UNSUPPORTED_FS_POLICY, RUSTFS_UNSUPPORTED_FS_POLICY_FAIL,
RUSTFS_UNSUPPORTED_FS_POLICY_WARN, RUSTFS_UNSUPPORTED_FS_POLICY_WARN,
+1 -1
View File
@@ -14,7 +14,7 @@
use crate::app::context::{AppContext, get_global_app_context, init_global_app_context}; use crate::app::context::{AppContext, get_global_app_context, init_global_app_context};
use crate::server::{ServiceStateManager, publish_ready_when_runtime_ready}; use crate::server::{ServiceStateManager, publish_ready_when_runtime_ready};
use crate::storage_compat::ecstore::store::ECStore; use crate::storage_compat::ECStore;
use rustfs_common::{GlobalReadiness, SystemStage}; use rustfs_common::{GlobalReadiness, SystemStage};
use rustfs_iam::init_iam_sys; use rustfs_iam::init_iam_sys;
use rustfs_kms::KmsServiceManager; use rustfs_kms::KmsServiceManager;
+2 -2
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::storage_compat::ecstore::global::set_global_rustfs_port; use crate::storage_compat::set_global_rustfs_port;
use crate::{ use crate::{
capacity::capacity_integration::init_capacity_management, capacity::capacity_integration::init_capacity_management,
config::Config, config::Config,
@@ -57,7 +57,7 @@ pub async fn init_startup_listen_context(config: &Config) -> Result<StartupListe
if let Some(region_str) = &config.region { if let Some(region_str) = &config.region {
region_str region_str
.parse::<s3s::region::Region>() .parse::<s3s::region::Region>()
.map(crate::storage_compat::ecstore::global::set_global_region) .map(crate::storage_compat::set_global_region)
.map_err(|err| Error::other(format!("invalid region '{}': {}", region_str, err)))?; .map_err(|err| Error::other(format!("invalid region '{}': {}", region_str, err)))?;
} }
+7 -17
View File
@@ -12,16 +12,9 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::storage_compat::ecstore::{ use crate::storage_compat::{
bucket::{ ECStore, EndpointServerPools, get_global_replication_pool, init_bucket_metadata_sys, new_global_notification_sys,
metadata_sys::init_bucket_metadata_sys, shutdown_background_services, try_migrate_bucket_metadata, try_migrate_iam_config,
migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
replication::get_global_replication_pool,
},
endpoints::EndpointServerPools,
global::shutdown_background_services,
notification_sys::new_global_notification_sys,
store::ECStore,
}; };
use crate::{ use crate::{
config::Config, config::Config,
@@ -597,16 +590,14 @@ where
start_audit().await start_audit().await
} }
pub async fn init_notification_system(endpoint_pools: EndpointServerPools) -> crate::storage_compat::ecstore::error::Result<()> { pub async fn init_notification_system(endpoint_pools: EndpointServerPools) -> crate::storage_compat::EcstoreResult<()> {
init_notification_system_with(|| new_global_notification_sys(endpoint_pools)).await init_notification_system_with(|| new_global_notification_sys(endpoint_pools)).await
} }
async fn init_notification_system_with<InitFn, InitFuture>( async fn init_notification_system_with<InitFn, InitFuture>(init_notification: InitFn) -> crate::storage_compat::EcstoreResult<()>
init_notification: InitFn,
) -> crate::storage_compat::ecstore::error::Result<()>
where where
InitFn: FnOnce() -> InitFuture, InitFn: FnOnce() -> InitFuture,
InitFuture: Future<Output = crate::storage_compat::ecstore::error::Result<()>>, InitFuture: Future<Output = crate::storage_compat::EcstoreResult<()>>,
{ {
init_notification().await init_notification().await
} }
@@ -664,8 +655,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn notification_system_returns_source_error() { async fn notification_system_returns_source_error() {
let result = let result = init_notification_system_with(|| async { Err(crate::storage_compat::EcstoreError::FaultyDisk) }).await;
init_notification_system_with(|| async { Err(crate::storage_compat::ecstore::error::Error::FaultyDisk) }).await;
assert!(result.is_err()); assert!(result.is_err());
} }
+6 -9
View File
@@ -13,12 +13,9 @@
// limitations under the License. // limitations under the License.
use crate::startup_fs_guard::enforce_unsupported_fs_policy; use crate::startup_fs_guard::enforce_unsupported_fs_policy;
use crate::storage_compat::ecstore::{ use crate::storage_compat::{
bucket::replication::init_background_replication, ECStore, EndpointServerPools, init as init_ecstore_config, init_background_replication, init_global_config_sys,
config as ecconfig, init_local_disks, init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints, try_migrate_server_config,
endpoints::EndpointServerPools,
set_global_endpoints,
store::{ECStore, init_local_disks, init_lock_clients, prewarm_local_disk_id_map},
update_erasure_type, update_erasure_type,
}; };
use rustfs_common::{GlobalReadiness, SystemStage}; use rustfs_common::{GlobalReadiness, SystemStage};
@@ -147,11 +144,11 @@ pub async fn init_startup_storage_runtime(
} }
async fn init_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> { async fn init_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> {
ecconfig::init(); init_ecstore_config();
ecconfig::try_migrate_server_config(store.clone()).await; try_migrate_server_config(store.clone()).await;
let mut retry_count = 0; let mut retry_count = 0;
while let Err(e) = ecconfig::init_global_config_sys(store.clone()).await { while let Err(e) = init_global_config_sys(store.clone()).await {
let next_retry_count = retry_count + 1; let next_retry_count = retry_count + 1;
error!( error!(
target: "rustfs::main::run", target: "rustfs::main::run",
+6 -6
View File
@@ -18,11 +18,11 @@ use crate::error::ApiError;
use crate::license::license_check; use crate::license::license_check;
use crate::server::RemoteAddr; use crate::server::RemoteAddr;
use crate::storage::request_context::RequestContext; use crate::storage::request_context::RequestContext;
use crate::storage::storage_compat::ecstore::bucket::metadata_sys; use crate::storage::storage_compat::ECStore;
use crate::storage::storage_compat::ecstore::bucket::policy_sys::PolicySys; use crate::storage::storage_compat::metadata_sys;
use crate::storage::storage_compat::ecstore::error::{StorageError, is_err_bucket_not_found}; use crate::storage::storage_compat::policy_sys::PolicySys;
use crate::storage::storage_compat::ecstore::resolve_object_store_handle; use crate::storage::storage_compat::resolve_object_store_handle;
use crate::storage::storage_compat::ecstore::store::ECStore; use crate::storage::storage_compat::{StorageError, is_err_bucket_not_found};
use metrics::counter; use metrics::counter;
use rustfs_iam::error::Error as IamError; use rustfs_iam::error::Error as IamError;
use rustfs_policy::policy::action::{Action, AdminAction, S3Action}; use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
@@ -930,7 +930,7 @@ impl S3Access for FS {
let req_info = ReqInfo { let req_info = ReqInfo {
cred, cred,
is_owner, is_owner,
region: crate::storage::storage_compat::ecstore::global::get_global_region(), region: crate::storage::storage_compat::get_global_region(),
request_context, request_context,
..Default::default() ..Default::default()
}; };
+12 -14
View File
@@ -21,21 +21,19 @@ use crate::storage::access::has_bypass_governance_header;
use crate::storage::helper::OperationHelper; use crate::storage::helper::OperationHelper;
use crate::storage::options::get_opts; use crate::storage::options::get_opts;
use crate::storage::s3_api::acl; use crate::storage::s3_api::acl;
use crate::storage::storage_compat::ecstore::{ use crate::storage::storage_compat::{
bucket::{ StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
metadata::{ metadata::{
BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG, BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG,
BUCKET_WEBSITE_CONFIG, OBJECT_LOCK_CONFIG, BUCKET_WEBSITE_CONFIG, OBJECT_LOCK_CONFIG,
},
metadata_sys,
object_lock::objectlock_sys::check_retention_for_modification,
replication::{GLOBAL_REPLICATION_STATS, ReplicationConfigurationExt},
tagging::{decode_tags, decode_tags_to_map, encode_tags},
utils::serialize,
versioning::VersioningApi,
versioning_sys::BucketVersioningSys,
}, },
error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found}, metadata_sys,
object_lock::objectlock_sys::check_retention_for_modification,
replication::{GLOBAL_REPLICATION_STATS, ReplicationConfigurationExt},
tagging::{decode_tags, decode_tags_to_map, encode_tags},
utils::serialize,
versioning::VersioningApi,
versioning_sys::BucketVersioningSys,
}; };
use crate::storage::{parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled}; use crate::storage::{parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled};
use crate::table_catalog; use crate::table_catalog;
+7 -7
View File
@@ -16,12 +16,12 @@ use crate::config::{RustFSBufferConfig, WorkloadProfile, get_global_buffer_confi
use crate::error::ApiError; use crate::error::ApiError;
use crate::server::cors; use crate::server::cors;
use crate::storage::ecfs::ListObjectUnorderedQuery; use crate::storage::ecfs::ListObjectUnorderedQuery;
use crate::storage::storage_compat::ecstore::bucket::metadata_sys; use crate::storage::storage_compat::StorageError;
use crate::storage::storage_compat::ecstore::bucket::metadata_sys::get_replication_config; use crate::storage::storage_compat::metadata_sys;
use crate::storage::storage_compat::ecstore::bucket::object_lock::objectlock_sys; use crate::storage::storage_compat::metadata_sys::get_replication_config;
use crate::storage::storage_compat::ecstore::bucket::replication::ReplicationConfigurationExt; use crate::storage::storage_compat::object_lock::objectlock_sys;
use crate::storage::storage_compat::ecstore::error::StorageError; use crate::storage::storage_compat::replication::ReplicationConfigurationExt;
use crate::storage::storage_compat::ecstore::resolve_object_store_handle; use crate::storage::storage_compat::resolve_object_store_handle;
use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE}; use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE};
use http::{HeaderMap, HeaderValue, StatusCode}; use http::{HeaderMap, HeaderValue, StatusCode};
use metrics::counter; use metrics::counter;
@@ -739,7 +739,7 @@ pub(crate) async fn has_replication_rules(bucket: &str, objects: &[ObjectToDelet
} }
/// Helper function to get store and validate bucket exists /// Helper function to get store and validate bucket exists
pub(crate) async fn get_validated_store(bucket: &str) -> S3Result<Arc<crate::storage::storage_compat::ecstore::store::ECStore>> { pub(crate) async fn get_validated_store(bucket: &str) -> S3Result<Arc<crate::storage::storage_compat::ECStore>> {
let Some(store) = resolve_object_store_handle() else { let Some(store) = resolve_object_store_handle() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())); return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
}; };
+6 -6
View File
@@ -18,8 +18,8 @@ mod tests {
use crate::server::cors; use crate::server::cors;
use crate::storage::ecfs::{FS, validate_object_lock_configuration_input}; use crate::storage::ecfs::{FS, validate_object_lock_configuration_input};
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
use crate::storage::storage_compat::ecstore::bucket::{metadata::BucketMetadata, metadata_sys}; use crate::storage::storage_compat::DEFAULT_READ_BUFFER_SIZE;
use crate::storage::storage_compat::ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; use crate::storage::storage_compat::{metadata::BucketMetadata, metadata_sys};
use crate::storage::{ use crate::storage::{
StorageObjectInfo as ObjectInfo, apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions, StorageObjectInfo as ObjectInfo, apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions,
get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal, matches_origin_pattern, parse_etag, get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal, matches_origin_pattern, parse_etag,
@@ -940,12 +940,12 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_validate_bucket_object_lock_enabled() { async fn test_validate_bucket_object_lock_enabled() {
use crate::storage::storage_compat::ecstore::bucket::metadata::BucketMetadata; use crate::storage::storage_compat::metadata::BucketMetadata;
use crate::storage::storage_compat::ecstore::bucket::metadata_sys::set_bucket_metadata; use crate::storage::storage_compat::metadata_sys::set_bucket_metadata;
use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled}; use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
use time::OffsetDateTime; use time::OffsetDateTime;
if crate::storage::storage_compat::ecstore::bucket::metadata_sys::GLOBAL_BucketMetadataSys if crate::storage::storage_compat::metadata_sys::GLOBAL_BucketMetadataSys
.get() .get()
.is_none() .is_none()
{ {
@@ -1783,7 +1783,7 @@ mod tests {
/// with a single-element vec value, matching the format expected by policy evaluation. /// with a single-element vec value, matching the format expected by policy evaluation.
#[test] #[test]
fn test_object_tag_condition_key_format() { fn test_object_tag_condition_key_format() {
use crate::storage::storage_compat::ecstore::bucket::tagging::decode_tags_to_map; use crate::storage::storage_compat::tagging::decode_tags_to_map;
use std::collections::HashMap; use std::collections::HashMap;
let tags_str = "security=public&project=webapp&env=prod"; let tags_str = "security=public&project=webapp&env=prod";
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::storage::storage_compat::ecstore::store::ECStore; use crate::storage::storage_compat::ECStore;
use rustfs_storage_api::ListOperations as _; use rustfs_storage_api::ListOperations as _;
use std::sync::Arc; use std::sync::Arc;
+4 -4
View File
@@ -29,11 +29,11 @@ pub mod timeout_wrapper;
pub mod tonic_service; pub mod tonic_service;
pub(crate) type StorageDeletedObject = rustfs_storage_api::DeletedObject; pub(crate) type StorageDeletedObject = rustfs_storage_api::DeletedObject;
pub(crate) type StorageGetObjectReader = crate::storage::storage_compat::ecstore::GetObjectReader; pub(crate) type StorageGetObjectReader = crate::storage::storage_compat::GetObjectReader;
pub(crate) type StorageObjectInfo = crate::storage::storage_compat::ecstore::ObjectInfo; pub(crate) type StorageObjectInfo = crate::storage::storage_compat::ObjectInfo;
pub(crate) type StorageObjectOptions = crate::storage::storage_compat::ecstore::ObjectOptions; pub(crate) type StorageObjectOptions = crate::storage::storage_compat::ObjectOptions;
pub(crate) type StorageObjectToDelete = rustfs_storage_api::ObjectToDelete; pub(crate) type StorageObjectToDelete = rustfs_storage_api::ObjectToDelete;
pub(crate) type StoragePutObjReader = crate::storage::storage_compat::ecstore::PutObjReader; pub(crate) type StoragePutObjReader = crate::storage::storage_compat::PutObjReader;
#[cfg(test)] #[cfg(test)]
mod concurrent_fix_test; mod concurrent_fix_test;
+3 -3
View File
@@ -12,9 +12,9 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use crate::storage::storage_compat::ecstore::bucket::versioning_sys::BucketVersioningSys; use crate::storage::storage_compat::Result;
use crate::storage::storage_compat::ecstore::error::Result; use crate::storage::storage_compat::StorageError;
use crate::storage::storage_compat::ecstore::error::StorageError; use crate::storage::storage_compat::versioning_sys::BucketVersioningSys;
use http::header::{IF_MATCH, IF_NONE_MATCH}; use http::header::{IF_MATCH, IF_NONE_MATCH};
use http::{HeaderMap, HeaderValue}; use http::{HeaderMap, HeaderValue};
use rustfs_utils::http::{ use rustfs_utils::http::{
+4 -4
View File
@@ -14,10 +14,10 @@
use crate::server::RPC_PREFIX; use crate::server::RPC_PREFIX;
use crate::storage::request_context::spawn_traced; use crate::storage::request_context::spawn_traced;
use crate::storage::storage_compat::ecstore::disk::{DiskAPI, WalkDirOptions}; use crate::storage::storage_compat::DEFAULT_READ_BUFFER_SIZE;
use crate::storage::storage_compat::ecstore::rpc::verify_rpc_signature; use crate::storage::storage_compat::find_local_disk_by_ref;
use crate::storage::storage_compat::ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; use crate::storage::storage_compat::verify_rpc_signature;
use crate::storage::storage_compat::ecstore::store::find_local_disk_by_ref; use crate::storage::storage_compat::{DiskAPI, WalkDirOptions};
use bytes::{Bytes, BytesMut}; use bytes::{Bytes, BytesMut};
use futures_util::TryStreamExt; use futures_util::TryStreamExt;
use http::{HeaderMap, Method, Request, Response, StatusCode, Uri}; use http::{HeaderMap, Method, Request, Response, StatusCode, Uri};
+10 -23
View File
@@ -16,22 +16,12 @@ use crate::admin::service::{
config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot}, config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot},
site_replication::reload_site_replication_runtime_state, site_replication::reload_site_replication_runtime_state,
}; };
use crate::storage::storage_compat::ecstore::{ use crate::storage::storage_compat::{
admin_server_info::get_local_server_property, CollectMetricsOpts, DeleteOptions, DiskAPI, DiskError, DiskInfoOptions, DiskStore, FileInfoVersions, GLOBAL_TierConfigMgr,
bucket::{metadata::load_bucket_metadata, metadata_sys}, LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, ReadMultipleReq, ReadMultipleResp,
disk::{ ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, UpdateMetadataOpts, all_local_disk_path,
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, collect_local_metrics, find_local_disk_by_ref, get_global_lock_client, get_local_server_property,
UpdateMetadataOpts, error::DiskError, metadata::load_bucket_metadata, metadata_sys, resolve_object_store_handle,
},
get_global_lock_client,
global::GLOBAL_TierConfigMgr,
metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics},
resolve_object_store_handle,
rpc::{
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC,
},
store::{all_local_disk_path, find_local_disk_by_ref},
}; };
use bytes::Bytes; use bytes::Bytes;
use futures::Stream; use futures::Stream;
@@ -132,9 +122,7 @@ fn unimplemented_rpc(method: &str) -> Status {
Status::unimplemented(format!("{method} is not implemented")) Status::unimplemented(format!("{method} is not implemented"))
} }
fn background_rebalance_start_error_message( fn background_rebalance_start_error_message(result: crate::storage::storage_compat::Result<()>) -> Option<String> {
result: crate::storage::storage_compat::ecstore::error::Result<()>,
) -> Option<String> {
result.err().map(|err| format!("start_rebalance failed: {err}")) result.err().map(|err| format!("start_rebalance failed: {err}"))
} }
@@ -2378,9 +2366,8 @@ mod tests {
#[test] #[test]
fn test_background_rebalance_start_error_message_formats_error() { fn test_background_rebalance_start_error_message_formats_error() {
let message = let message = background_rebalance_start_error_message(Err(crate::storage::storage_compat::Error::other("boom")))
background_rebalance_start_error_message(Err(crate::storage::storage_compat::ecstore::error::Error::other("boom"))) .expect("background rebalance start failure should be formatted");
.expect("background rebalance start failure should be formatted");
assert!(message.contains("start_rebalance failed")); assert!(message.contains("start_rebalance failed"));
assert!(message.contains("boom")); assert!(message.contains("boom"));
@@ -2710,7 +2697,7 @@ mod tests {
vars.insert(PEER_RESTSIGNAL.to_string(), SERVICE_SIGNAL_RELOAD_DYNAMIC.to_string()); vars.insert(PEER_RESTSIGNAL.to_string(), SERVICE_SIGNAL_RELOAD_DYNAMIC.to_string());
vars.insert( vars.insert(
PEER_RESTSUB_SYS.to_string(), PEER_RESTSUB_SYS.to_string(),
crate::storage::storage_compat::ecstore::config::com::STORAGE_CLASS_SUB_SYS.to_string(), crate::storage::storage_compat::com::STORAGE_CLASS_SUB_SYS.to_string(),
); );
let request = Request::new(SignalServiceRequest { let request = Request::new(SignalServiceRequest {
+1 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License. // limitations under the License.
use crate::storage::s3_api::common::rustfs_owner; use crate::storage::s3_api::common::rustfs_owner;
use crate::storage::storage_compat::ecstore::client::object_api_utils::to_s3s_etag; use crate::storage::storage_compat::object_api_utils::to_s3s_etag;
use percent_encoding::percent_decode_str; use percent_encoding::percent_decode_str;
use rustfs_storage_api::{ use rustfs_storage_api::{
BucketInfo, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, BucketInfo, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
+2 -2
View File
@@ -13,7 +13,7 @@
// limitations under the License. // limitations under the License.
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
use crate::storage::storage_compat::ecstore::client::object_api_utils::to_s3s_etag; use crate::storage::storage_compat::object_api_utils::to_s3s_etag;
use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo}; use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo};
use s3s::dto::{CommonPrefix, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp}; use s3s::dto::{CommonPrefix, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp};
use s3s::{S3Error, S3ErrorCode}; use s3s::{S3Error, S3ErrorCode};
@@ -191,7 +191,7 @@ mod tests {
parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number, parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
}; };
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
use crate::storage::storage_compat::ecstore::client::object_api_utils::to_s3s_etag; use crate::storage::storage_compat::object_api_utils::to_s3s_etag;
use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo}; use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo};
use s3s::S3ErrorCode; use s3s::S3ErrorCode;
use s3s::dto::Timestamp; use s3s::dto::Timestamp;
+8 -18
View File
@@ -69,7 +69,7 @@
//! } //! }
//! ``` //! ```
use crate::storage::storage_compat::ecstore::error::StorageError; use crate::storage::storage_compat::StorageError;
use aes_gcm::{ use aes_gcm::{
Aes256Gcm, Key, Nonce, Aes256Gcm, Key, Nonce,
aead::{Aead, KeyInit}, aead::{Aead, KeyInit},
@@ -128,8 +128,8 @@ const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE;
const OBJECT_KEY_DERIVATION_CONTEXT: &[u8] = b"object-encryption-key generation"; const OBJECT_KEY_DERIVATION_CONTEXT: &[u8] = b"object-encryption-key generation";
use crate::error::ApiError; use crate::error::ApiError;
use crate::storage::storage_compat::ecstore::bucket::metadata_sys; use crate::storage::storage_compat::Error;
use crate::storage::storage_compat::ecstore::error::Error; use crate::storage::storage_compat::metadata_sys;
use rustfs_utils::http::headers::{ use rustfs_utils::http::headers::{
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT,
@@ -745,29 +745,19 @@ pub struct ManagedSealedKey {
} }
impl EncryptionMaterial { impl EncryptionMaterial {
pub fn write_encryption( pub fn write_encryption(&self, multipart_part_number: Option<usize>) -> crate::storage::storage_compat::WriteEncryption {
&self,
multipart_part_number: Option<usize>,
) -> crate::storage::storage_compat::ecstore::rio::WriteEncryption {
match (self.key_kind, multipart_part_number) { match (self.key_kind, multipart_part_number) {
(EncryptionKeyKind::Object, Some(part_number)) => { (EncryptionKeyKind::Object, Some(part_number)) => {
crate::storage::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key( crate::storage::storage_compat::WriteEncryption::multipart_object_key(self.key_bytes, part_number as u32)
self.key_bytes,
part_number as u32,
)
} }
(EncryptionKeyKind::Object, None) => { (EncryptionKeyKind::Object, None) => {
crate::storage::storage_compat::ecstore::rio::WriteEncryption::singlepart_object_key(self.key_bytes) crate::storage::storage_compat::WriteEncryption::singlepart_object_key(self.key_bytes)
} }
(EncryptionKeyKind::Direct, Some(part_number)) => { (EncryptionKeyKind::Direct, Some(part_number)) => {
crate::storage::storage_compat::ecstore::rio::WriteEncryption::multipart( crate::storage::storage_compat::WriteEncryption::multipart(self.key_bytes, self.base_nonce, part_number)
self.key_bytes,
self.base_nonce,
part_number,
)
} }
(EncryptionKeyKind::Direct, None) => { (EncryptionKeyKind::Direct, None) => {
crate::storage::storage_compat::ecstore::rio::WriteEncryption::singlepart(self.key_bytes, self.base_nonce) crate::storage::storage_compat::WriteEncryption::singlepart(self.key_bytes, self.base_nonce)
} }
} }
} }
+30 -66
View File
@@ -12,70 +12,34 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
pub(crate) mod ecstore { pub(crate) use rustfs_ecstore::admin_server_info::get_local_server_property;
pub(crate) use rustfs_ecstore::global::{get_global_lock_client, resolve_object_store_handle}; pub(crate) use rustfs_ecstore::bucket::{
metadata, metadata_sys, object_lock, policy_sys, replication, tagging, utils, versioning, versioning_sys,
};
pub(crate) use rustfs_ecstore::client::object_api_utils;
#[cfg(test)]
pub(crate) use rustfs_ecstore::config::com;
pub(crate) use rustfs_ecstore::disk::error::DiskError;
pub(crate) use rustfs_ecstore::disk::{
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
UpdateMetadataOpts, WalkDirOptions,
};
pub(crate) use rustfs_ecstore::error::{
Error, Result, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
pub(crate) use rustfs_ecstore::global::{
GLOBAL_TierConfigMgr, get_global_lock_client, get_global_region, resolve_object_store_handle,
};
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
pub(crate) use rustfs_ecstore::rio::WriteEncryption;
pub(crate) use rustfs_ecstore::rpc::{
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, verify_rpc_signature,
};
pub(crate) use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk_path, find_local_disk_by_ref};
pub(crate) mod admin_server_info { pub(crate) type GetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
pub(crate) use rustfs_ecstore::admin_server_info::get_local_server_property; pub(crate) type ObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
} pub(crate) type ObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub(crate) type PutObjReader = rustfs_ecstore::store_api::PutObjReader;
pub(crate) mod bucket {
pub(crate) use rustfs_ecstore::bucket::{
metadata, metadata_sys, object_lock, policy_sys, replication, tagging, utils, versioning, versioning_sys,
};
}
pub(crate) mod client {
pub(crate) use rustfs_ecstore::client::object_api_utils;
}
#[cfg(test)]
pub(crate) mod config {
pub(crate) use rustfs_ecstore::config::com;
}
pub(crate) mod disk {
pub(crate) use rustfs_ecstore::disk::{
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
UpdateMetadataOpts, WalkDirOptions, error,
};
}
pub(crate) mod error {
pub(crate) use rustfs_ecstore::error::{
Error, Result, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
}
pub(crate) mod global {
pub(crate) use rustfs_ecstore::global::{GLOBAL_TierConfigMgr, get_global_region};
}
pub(crate) mod metrics_realtime {
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
}
pub(crate) mod rio {
pub(crate) use rustfs_ecstore::rio::WriteEncryption;
}
pub(crate) mod rpc {
pub(crate) use rustfs_ecstore::rpc::{
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, verify_rpc_signature,
};
}
pub(crate) mod set_disk {
pub(crate) use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
}
pub(crate) mod store {
pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk_path, find_local_disk_by_ref};
}
pub(crate) type GetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
pub(crate) type ObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
pub(crate) type ObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
pub(crate) type PutObjReader = rustfs_ecstore::store_api::PutObjReader;
}
+21 -59
View File
@@ -12,62 +12,24 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
pub(crate) mod ecstore { pub(crate) use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
pub(crate) use rustfs_ecstore::global::{resolve_object_store_handle, set_global_endpoints, update_erasure_type}; pub(crate) use rustfs_ecstore::bucket::migration::{try_migrate_bucket_metadata, try_migrate_iam_config};
pub(crate) use rustfs_ecstore::bucket::replication::{get_global_replication_pool, init_background_replication};
pub(crate) mod bucket { pub(crate) use rustfs_ecstore::bucket::{metadata, metadata_sys, quota};
pub(crate) use rustfs_ecstore::bucket::{metadata, metadata_sys, migration, quota, replication}; pub(crate) use rustfs_ecstore::config::{com, init, init_global_config_sys, try_migrate_server_config};
} pub(crate) use rustfs_ecstore::disk::{DiskAPI, RUSTFS_META_BUCKET, endpoint::Endpoint};
#[cfg(test)]
pub(crate) mod config { pub(crate) use rustfs_ecstore::disks_layout::DisksLayout;
pub(crate) use rustfs_ecstore::config::{com, init, init_global_config_sys, try_migrate_server_config}; pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
} #[cfg(test)]
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
pub(crate) mod disk { pub(crate) use rustfs_ecstore::error::{Error as EcstoreError, Result as EcstoreResult, StorageError};
pub(crate) use rustfs_ecstore::disk::{DiskAPI, RUSTFS_META_BUCKET, endpoint}; pub(crate) use rustfs_ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook};
} pub(crate) use rustfs_ecstore::global::{
get_global_endpoints_opt, get_global_lock_clients, get_global_region, is_dist_erasure, resolve_object_store_handle,
#[cfg(test)] set_global_endpoints, set_global_region, set_global_rustfs_port, shutdown_background_services, update_erasure_type,
pub(crate) mod disks_layout { };
pub(crate) use rustfs_ecstore::disks_layout::DisksLayout; pub(crate) use rustfs_ecstore::notification_sys::new_global_notification_sys;
} pub(crate) use rustfs_ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
pub(crate) use rustfs_ecstore::set_disk::get_lock_acquire_timeout;
pub(crate) mod endpoints { pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk, init_local_disks, init_lock_clients, prewarm_local_disk_id_map};
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
#[cfg(test)]
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
}
pub(crate) mod error {
pub(crate) use rustfs_ecstore::error::{Error, Result, StorageError};
}
pub(crate) mod event_notification {
pub(crate) use rustfs_ecstore::event_notification::{EventArgs, register_event_dispatch_hook};
}
pub(crate) mod global {
pub(crate) use rustfs_ecstore::global::{
get_global_endpoints_opt, get_global_lock_clients, get_global_region, is_dist_erasure, set_global_region,
set_global_rustfs_port, shutdown_background_services,
};
}
pub(crate) mod notification_sys {
pub(crate) use rustfs_ecstore::notification_sys::new_global_notification_sys;
}
pub(crate) mod rpc {
pub(crate) use rustfs_ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
}
pub(crate) mod set_disk {
pub(crate) use rustfs_ecstore::set_disk::get_lock_acquire_timeout;
}
pub(crate) mod store {
pub(crate) use rustfs_ecstore::store::{
ECStore, all_local_disk, init_local_disks, init_lock_clients, prewarm_local_disk_id_map,
};
}
}
+12 -12
View File
@@ -27,16 +27,16 @@ use std::{
time::{Duration as StdDuration, Instant}, time::{Duration as StdDuration, Instant},
}; };
use crate::storage_compat::ecstore::bucket::{ use crate::storage_compat::RUSTFS_META_BUCKET;
use crate::storage_compat::get_lock_acquire_timeout;
use crate::storage_compat::{EcstoreError, StorageError};
use crate::storage_compat::{
metadata::{ metadata::{
BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG, BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG,
BUCKET_TABLE_RESERVED_PREFIX, table_catalog_path_hash, BUCKET_TABLE_RESERVED_PREFIX, table_catalog_path_hash,
}, },
metadata_sys, metadata_sys,
}; };
use crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET;
use crate::storage_compat::ecstore::error::{Error as EcstoreError, StorageError};
use crate::storage_compat::ecstore::set_disk::get_lock_acquire_timeout;
use bytes::Bytes; use bytes::Bytes;
use datafusion::{ use datafusion::{
arrow::datatypes::SchemaRef, arrow::datatypes::SchemaRef,
@@ -7813,7 +7813,7 @@ mod tests {
.map(|(bucket, _)| bucket.as_str()) .map(|(bucket, _)| bucket.as_str())
.collect::<BTreeSet<_>>(); .collect::<BTreeSet<_>>();
assert_eq!(object_buckets, BTreeSet::from([crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET])); assert_eq!(object_buckets, BTreeSet::from([crate::storage_compat::RUSTFS_META_BUCKET]));
} }
#[tokio::test] #[tokio::test]
@@ -10777,7 +10777,7 @@ mod tests {
.unwrap(); .unwrap();
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &idempotency_path, 1) .fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &idempotency_path, 1)
.await; .await;
let err = store let err = store
@@ -10827,7 +10827,7 @@ mod tests {
.unwrap(); .unwrap();
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2) .fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
.await; .await;
let request = TableCommitRequest { let request = TableCommitRequest {
@@ -10882,7 +10882,7 @@ mod tests {
.unwrap(); .unwrap();
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2) .fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
.await; .await;
let result = store let result = store
@@ -10936,7 +10936,7 @@ mod tests {
.unwrap(); .unwrap();
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2) .fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
.await; .await;
store store
@@ -10990,7 +10990,7 @@ mod tests {
.unwrap(); .unwrap();
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2) .fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
.await; .await;
store store
@@ -11041,7 +11041,7 @@ mod tests {
backend.seed_object(bucket, &current_metadata, b"{}".to_vec()).await; backend.seed_object(bucket, &current_metadata, b"{}".to_vec()).await;
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &table_path, 2) .fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &table_path, 2)
.await; .await;
let err = store let err = store
@@ -11098,7 +11098,7 @@ mod tests {
.unwrap(); .unwrap();
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await; backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
backend backend
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &idempotency_path, 2) .fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &idempotency_path, 2)
.await; .await;
let result = store let result = store
@@ -65,6 +65,10 @@ STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_list_c
STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_operation_consumer_hits.txt" STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_operation_consumer_hits.txt"
STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE="${TMP_DIR}/store_api_object_operation_local_method_hits.txt" STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE="${TMP_DIR}/store_api_object_operation_local_method_hits.txt"
DIRECT_ECSTORE_IMPORT_HITS_FILE="${TMP_DIR}/direct_ecstore_import_hits.txt" DIRECT_ECSTORE_IMPORT_HITS_FILE="${TMP_DIR}/direct_ecstore_import_hits.txt"
TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/test_harness_nested_storage_compat_hits.txt"
RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/rustfs_nested_storage_compat_hits.txt"
RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/rustfs_runtime_scalar_storage_compat_hits.txt"
RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/rustfs_runtime_secondary_storage_compat_hits.txt"
PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE="${TMP_DIR}/production_unused_compat_allow_hits.txt" PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE="${TMP_DIR}/production_unused_compat_allow_hits.txt"
BROAD_STORE_API_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/broad_store_api_compat_reexport_hits.txt" BROAD_STORE_API_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/broad_store_api_compat_reexport_hits.txt"
NESTED_STORE_API_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/nested_store_api_compat_module_hits.txt" NESTED_STORE_API_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/nested_store_api_compat_module_hits.txt"
@@ -204,10 +208,18 @@ require_source_line \
"crates/storage-api/src/lib.rs" \ "crates/storage-api/src/lib.rs" \
"pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};" \ "pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};" \
"storage-api public bucket contract re-export" "storage-api public bucket contract re-export"
require_source_line \
"crates/storage-api/src/lib.rs" \
"pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};" \
"storage-api public capability contract re-export"
require_source_line \ require_source_line \
"crates/storage-api/src/lib.rs" \ "crates/storage-api/src/lib.rs" \
"pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};" \ "pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};" \
"storage-api public multipart DTO re-export" "storage-api public multipart DTO re-export"
require_source_line \
"crates/storage-api/src/lib.rs" \
"pub use observability::{" \
"storage-api public observability contract re-export"
require_source_line \ require_source_line \
"crates/storage-api/src/lib.rs" \ "crates/storage-api/src/lib.rs" \
"pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};" \ "pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};" \
@@ -240,6 +252,10 @@ require_source_line \
"crates/storage-api/src/lib.rs" \ "crates/storage-api/src/lib.rs" \
"pub use error::{StorageErrorCode, StorageResult};" \ "pub use error::{StorageErrorCode, StorageResult};" \
"storage-api public error contract re-export" "storage-api public error contract re-export"
require_source_line \
"crates/storage-api/src/lib.rs" \
"pub use topology::{" \
"storage-api public topology contract re-export"
( (
cd "$ROOT_DIR" cd "$ROOT_DIR"
@@ -403,6 +419,66 @@ if [[ -s "$DIRECT_ECSTORE_IMPORT_HITS_FILE" ]]; then
report_failure "direct rustfs_ecstore imports outside compatibility boundaries are forbidden: $(paste -sd '; ' "$DIRECT_ECSTORE_IMPORT_HITS_FILE")" report_failure "direct rustfs_ecstore imports outside compatibility boundaries are forbidden: $(paste -sd '; ' "$DIRECT_ECSTORE_IMPORT_HITS_FILE")"
fi fi
(
cd "$ROOT_DIR"
rg -n --no-heading '\bstorage_compat::ecstore\b|\bmod\s+ecstore\b' \
crates/e2e_test/src crates/heal/tests crates/scanner/tests fuzz/fuzz_targets \
--glob '*.rs' \
--glob '!target/**' || true
) >"$TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE"
if [[ -s "$TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE" ]]; then
report_failure "test and fuzz storage compatibility harnesses must use direct aliases instead of nested ecstore modules: $(paste -sd '; ' "$TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --no-heading '\bstorage_compat::ecstore\b|\bmod\s+ecstore\b' \
rustfs/src \
--glob '*.rs' \
--glob '!target/**' || true
) >"$RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE"
if [[ -s "$RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE" ]]; then
report_failure "RustFS runtime storage compatibility paths must use direct aliases instead of nested ecstore modules: $(paste -sd '; ' "$RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
rg -n --no-heading '\bstorage_compat::(?:store|error|global|endpoints|set_disk|rpc|metrics_realtime|notification_sys|admin_server_info|store_utils|data_usage|disks_layout|event_notification)::|\bstorage_compat::disk::(?:RUSTFS_META_BUCKET|DiskAPI|endpoint::Endpoint)\b|\bstorage_compat::\{[^}\n]*(?:store|error|global|endpoints|set_disk|rpc|metrics_realtime|notification_sys|admin_server_info|store_utils|data_usage|disks_layout|event_notification|disk)::' \
rustfs/src \
--glob '*.rs' \
--glob '!target/**' || true
) >"$RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE"
if [[ -s "$RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE" ]]; then
report_failure "RustFS runtime scalar storage compatibility paths must use direct aliases instead of secondary modules: $(paste -sd '; ' "$RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE")"
fi
(
cd "$ROOT_DIR"
{
rg -n --no-heading '\bstorage_compat::(?:bucket|config|rio|client|tier|compress|disk|rebalance)::' \
rustfs/src \
--glob '*.rs' \
--glob '!target/**' || true
rg -n --no-heading '\bpub\(crate\)\s+mod\s+(?:bucket|config|rio|client|tier|compress|disk|rebalance)\b' \
rustfs/src \
--glob '*storage_compat.rs' \
--glob '!target/**' || true
find rustfs/src -type f -name '*.rs' -print0 |
xargs -0 perl -0ne '
while (/use\s+crate::(?:app::|admin::|storage::)?storage_compat::\{[^;]*?\b(bucket|config|rio|client|tier|compress|disk|rebalance)::/sg) {
print "$ARGV:grouped storage_compat::$1 import\n";
}
'
}
) >"$RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE"
if [[ -s "$RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE" ]]; then
report_failure "RustFS runtime secondary storage compatibility paths must use direct aliases instead of bucket/config/rio/client/tier/compress/disk/rebalance modules: $(paste -sd '; ' "$RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE")"
fi
( (
cd "$ROOT_DIR" cd "$ROOT_DIR"
rg -n --no-heading '#!\[allow\(unused_imports\)\]' \ rg -n --no-heading '#!\[allow\(unused_imports\)\]' \