mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-29 16:37:07 +00:00
refactor: flatten test harness storage compat aliases (#3596)
* refactor: flatten test harness storage compat aliases * refactor: flatten rustfs storage compat aliases (#3597) * refactor: prune runtime storage compat surface (#3598) * refactor: flatten runtime secondary storage compat (#3599) * docs: add scheduler placement profiling baselines (#3600) * feat: add observability topology capability contracts (#3601)
This commit is contained in:
@@ -15,7 +15,7 @@
|
|||||||
// Used by test_distributed_lock_4_nodes_grpc in lock.rs
|
// Used by test_distributed_lock_4_nodes_grpc in lock.rs
|
||||||
#![allow(dead_code)]
|
#![allow(dead_code)]
|
||||||
|
|
||||||
use crate::storage_compat::ecstore::rpc::node_service_time_out_client_no_auth;
|
use crate::storage_compat::node_service_time_out_client_no_auth;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use rustfs_lock::{
|
use rustfs_lock::{
|
||||||
LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
LockClient, LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||||
@@ -41,10 +41,7 @@ impl GrpcLockClient {
|
|||||||
&self,
|
&self,
|
||||||
) -> Result<
|
) -> Result<
|
||||||
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
|
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
|
||||||
tonic::service::interceptor::InterceptedService<
|
tonic::service::interceptor::InterceptedService<tonic::transport::Channel, crate::storage_compat::TonicInterceptor>,
|
||||||
tonic::transport::Channel,
|
|
||||||
crate::storage_compat::ecstore::rpc::TonicInterceptor,
|
|
||||||
>,
|
|
||||||
>,
|
>,
|
||||||
> {
|
> {
|
||||||
node_service_time_out_client_no_auth(&self.addr)
|
node_service_time_out_client_no_auth(&self.addr)
|
||||||
|
|||||||
@@ -14,8 +14,8 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::common::workspace_root;
|
use crate::common::workspace_root;
|
||||||
use crate::storage_compat::ecstore::disk::{VolumeInfo, WalkDirOptions};
|
use crate::storage_compat::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||||
use crate::storage_compat::ecstore::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
use crate::storage_compat::{VolumeInfo, WalkDirOptions};
|
||||||
use futures::future::join_all;
|
use futures::future::join_all;
|
||||||
use rmp_serde::{Deserializer, Serializer};
|
use rmp_serde::{Deserializer, Serializer};
|
||||||
use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
|
use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
|
||||||
|
|||||||
@@ -15,7 +15,7 @@
|
|||||||
use crate::common::{
|
use crate::common::{
|
||||||
RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
|
RustFSTestEnvironment, awscurl_available, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
|
||||||
};
|
};
|
||||||
use crate::storage_compat::ecstore::bucket::bucket_target_sys::BucketTargetSys;
|
use crate::storage_compat::BucketTargetSys;
|
||||||
use aws_sdk_s3::config::{Credentials, Region};
|
use aws_sdk_s3::config::{Credentials, Region};
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||||
|
|||||||
@@ -12,22 +12,13 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
#![allow(dead_code, unused_imports)]
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) mod bucket {
|
pub(crate) type BucketTargetSys = rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
|
||||||
pub(crate) mod bucket_target_sys {
|
pub(crate) type TonicInterceptor = rustfs_ecstore::rpc::TonicInterceptor;
|
||||||
pub(crate) use rustfs_ecstore::bucket::bucket_target_sys::BucketTargetSys;
|
pub(crate) type VolumeInfo = rustfs_ecstore::disk::VolumeInfo;
|
||||||
}
|
pub(crate) type WalkDirOptions = rustfs_ecstore::disk::WalkDirOptions;
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod disk {
|
pub(crate) use rustfs_ecstore::rpc::{
|
||||||
pub(crate) use rustfs_ecstore::disk::{VolumeInfo, WalkDirOptions};
|
gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
||||||
}
|
};
|
||||||
|
|
||||||
pub(crate) mod rpc {
|
|
||||||
pub(crate) use rustfs_ecstore::rpc::{
|
|
||||||
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -12,28 +12,10 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
#![allow(unused_imports)]
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) mod bucket {
|
pub(crate) use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
|
||||||
pub(crate) mod metadata_sys {
|
pub(crate) use rustfs_ecstore::disk::{DiskStore, endpoint::Endpoint};
|
||||||
pub(crate) use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
|
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||||
}
|
pub(crate) use rustfs_ecstore::store::ECStore;
|
||||||
}
|
pub(crate) use rustfs_ecstore::store::init_local_disks;
|
||||||
|
|
||||||
pub(crate) mod disk {
|
|
||||||
pub(crate) use rustfs_ecstore::disk::DiskStore;
|
|
||||||
|
|
||||||
pub(crate) mod endpoint {
|
|
||||||
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod endpoints {
|
|
||||||
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod store {
|
|
||||||
pub(crate) use rustfs_ecstore::store::{ECStore, init_local_disks};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -16,8 +16,7 @@
|
|||||||
|
|
||||||
mod common;
|
mod common;
|
||||||
|
|
||||||
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
|
use crate::common::storage_compat::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_local_disks};
|
||||||
use crate::common::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
@@ -73,12 +72,10 @@ async fn test_endpoint_index_settings() -> anyhow::Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// test ECStore initialization
|
// test ECStore initialization
|
||||||
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone()).await?;
|
init_local_disks(endpoint_pools.clone()).await?;
|
||||||
|
|
||||||
let server_addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
|
let server_addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||||
let ecstore =
|
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new()).await?;
|
||||||
crate::common::storage_compat::ecstore::store::ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
println!("ECStore initialized successfully with {} pools", ecstore.pools.len());
|
println!("ECStore initialized successfully with {} pools", ecstore.pools.len());
|
||||||
|
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ use rustfs_heal::heal::{
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_heal_event_to_heal_request_no_panic() {
|
fn test_heal_event_to_heal_request_no_panic() {
|
||||||
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
|
use crate::common::storage_compat::Endpoint;
|
||||||
|
|
||||||
// Test that invalid pool/set indices don't cause panic
|
// Test that invalid pool/set indices don't cause panic
|
||||||
// Create endpoint using try_from or similar method
|
// Create endpoint using try_from or similar method
|
||||||
@@ -47,7 +47,7 @@ fn test_heal_event_to_heal_request_no_panic() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_heal_event_to_heal_request_valid_indices() {
|
fn test_heal_event_to_heal_request_valid_indices() {
|
||||||
use crate::common::storage_compat::ecstore::disk::endpoint::Endpoint;
|
use crate::common::storage_compat::Endpoint;
|
||||||
|
|
||||||
// Test that valid indices work correctly
|
// Test that valid indices work correctly
|
||||||
let endpoint_result = Endpoint::try_from("http://localhost:9000");
|
let endpoint_result = Endpoint::try_from("http://localhost:9000");
|
||||||
@@ -192,14 +192,11 @@ fn test_heal_task_status_atomic_update() {
|
|||||||
}
|
}
|
||||||
async fn get_disk_status(
|
async fn get_disk_status(
|
||||||
&self,
|
&self,
|
||||||
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
|
_endpoint: &crate::common::storage_compat::Endpoint,
|
||||||
) -> rustfs_heal::Result<rustfs_heal::heal::storage::DiskStatus> {
|
) -> rustfs_heal::Result<rustfs_heal::heal::storage::DiskStatus> {
|
||||||
Ok(rustfs_heal::heal::storage::DiskStatus::Ok)
|
Ok(rustfs_heal::heal::storage::DiskStatus::Ok)
|
||||||
}
|
}
|
||||||
async fn format_disk(
|
async fn format_disk(&self, _endpoint: &crate::common::storage_compat::Endpoint) -> rustfs_heal::Result<()> {
|
||||||
&self,
|
|
||||||
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
|
|
||||||
) -> rustfs_heal::Result<()> {
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_storage_api::BucketInfo>> {
|
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<rustfs_storage_api::BucketInfo>> {
|
||||||
@@ -253,10 +250,7 @@ fn test_heal_task_status_atomic_update() {
|
|||||||
) -> rustfs_heal::Result<(Vec<String>, Option<String>, bool)> {
|
) -> rustfs_heal::Result<(Vec<String>, Option<String>, bool)> {
|
||||||
Ok((vec![], None, false))
|
Ok((vec![], None, false))
|
||||||
}
|
}
|
||||||
async fn get_disk_for_resume(
|
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<crate::common::storage_compat::DiskStore> {
|
||||||
&self,
|
|
||||||
_set_disk_id: &str,
|
|
||||||
) -> rustfs_heal::Result<crate::common::storage_compat::ecstore::disk::DiskStore> {
|
|
||||||
Err(rustfs_heal::Error::other("Not implemented in mock"))
|
Err(rustfs_heal::Error::other("Not implemented in mock"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -326,17 +320,11 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
|
|||||||
Ok(Vec::new())
|
Ok(Vec::new())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_disk_status(
|
async fn get_disk_status(&self, _endpoint: &crate::common::storage_compat::Endpoint) -> rustfs_heal::Result<DiskStatus> {
|
||||||
&self,
|
|
||||||
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
|
|
||||||
) -> rustfs_heal::Result<DiskStatus> {
|
|
||||||
Ok(DiskStatus::Ok)
|
Ok(DiskStatus::Ok)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn format_disk(
|
async fn format_disk(&self, _endpoint: &crate::common::storage_compat::Endpoint) -> rustfs_heal::Result<()> {
|
||||||
&self,
|
|
||||||
_endpoint: &crate::common::storage_compat::ecstore::disk::endpoint::Endpoint,
|
|
||||||
) -> rustfs_heal::Result<()> {
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -406,10 +394,7 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
|
|||||||
Ok((Vec::new(), None, false))
|
Ok((Vec::new(), None, false))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_disk_for_resume(
|
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> rustfs_heal::Result<crate::common::storage_compat::DiskStore> {
|
||||||
&self,
|
|
||||||
_set_disk_id: &str,
|
|
||||||
) -> rustfs_heal::Result<crate::common::storage_compat::ecstore::disk::DiskStore> {
|
|
||||||
Err(rustfs_heal::Error::other("not implemented"))
|
Err(rustfs_heal::Error::other("not implemented"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,10 +14,8 @@
|
|||||||
|
|
||||||
mod common;
|
mod common;
|
||||||
|
|
||||||
use crate::common::storage_compat::ecstore::{
|
use crate::common::storage_compat::{
|
||||||
disk::endpoint::Endpoint,
|
ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, init_bucket_metadata_sys, init_local_disks,
|
||||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
|
||||||
store::ECStore,
|
|
||||||
};
|
};
|
||||||
use http::HeaderMap;
|
use http::HeaderMap;
|
||||||
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
|
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
|
||||||
@@ -124,9 +122,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
|
|||||||
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
||||||
|
|
||||||
// format disks (only first time)
|
// format disks (only first time)
|
||||||
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
|
init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// create ECStore with dynamic port 0 (let OS assign) or fixed 9001 if free
|
// create ECStore with dynamic port 0 (let OS assign) or fixed 9001 if free
|
||||||
let port = 9001; // for simplicity
|
let port = 9001; // for simplicity
|
||||||
@@ -144,7 +140,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>, Arc<ECStoreHealStorage
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||||
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||||
|
|
||||||
// Create heal storage layer
|
// Create heal storage layer
|
||||||
let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
|
let heal_storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
|
||||||
|
|||||||
@@ -12,70 +12,27 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
#![allow(dead_code, unused_imports)]
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) mod bucket {
|
pub(crate) const BUCKET_LIFECYCLE_CONFIG: &str = rustfs_ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
||||||
pub(crate) mod lifecycle {
|
pub(crate) const STORAGE_FORMAT_FILE: &str = rustfs_ecstore::disk::STORAGE_FORMAT_FILE;
|
||||||
pub(crate) use rustfs_ecstore::bucket::lifecycle::lifecycle::TransitionOptions;
|
|
||||||
|
|
||||||
pub(crate) mod bucket_lifecycle_ops {
|
pub(crate) type TransitionOptions = rustfs_ecstore::bucket::lifecycle::lifecycle::TransitionOptions;
|
||||||
pub(crate) use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{
|
|
||||||
enqueue_transition_for_existing_objects, init_background_expiry,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod metadata {
|
pub(crate) use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::{
|
||||||
pub(crate) use rustfs_ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
|
enqueue_transition_for_existing_objects, init_background_expiry,
|
||||||
}
|
};
|
||||||
|
pub(crate) use rustfs_ecstore::bucket::metadata_sys::{
|
||||||
pub(crate) mod metadata_sys {
|
get as get_bucket_metadata, init_bucket_metadata_sys, update as update_bucket_metadata,
|
||||||
pub(crate) use rustfs_ecstore::bucket::metadata_sys::{get, init_bucket_metadata_sys, update};
|
};
|
||||||
}
|
pub(crate) use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
|
||||||
|
pub(crate) use rustfs_ecstore::client::transition_api::{ReadCloser, ReaderImpl};
|
||||||
pub(crate) mod versioning_sys {
|
pub(crate) use rustfs_ecstore::disk::{DiskAPI, DiskOption, endpoint::Endpoint, new_disk};
|
||||||
pub(crate) use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys;
|
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||||
}
|
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
|
||||||
}
|
pub(crate) use rustfs_ecstore::pools::path2_bucket_object_with_base_path;
|
||||||
|
pub(crate) use rustfs_ecstore::store::ECStore;
|
||||||
pub(crate) mod client {
|
pub(crate) use rustfs_ecstore::store::init_local_disks;
|
||||||
pub(crate) mod transition_api {
|
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierMinIO, TierType};
|
||||||
pub(crate) use rustfs_ecstore::client::transition_api::{ReadCloser, ReaderImpl};
|
pub(crate) use rustfs_ecstore::tier::warm_backend::WarmBackendGetOpts;
|
||||||
}
|
pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, build_transition_put_options};
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod disk {
|
|
||||||
pub(crate) use rustfs_ecstore::disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, new_disk};
|
|
||||||
|
|
||||||
pub(crate) mod endpoint {
|
|
||||||
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod endpoints {
|
|
||||||
pub(crate) use rustfs_ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod global {
|
|
||||||
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod pools {
|
|
||||||
pub(crate) use rustfs_ecstore::pools::path2_bucket_object_with_base_path;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod store {
|
|
||||||
pub(crate) use rustfs_ecstore::store::{ECStore, init_local_disks};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod tier {
|
|
||||||
pub(crate) mod tier_config {
|
|
||||||
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierMinIO, TierType};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod warm_backend {
|
|
||||||
pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -14,24 +14,12 @@
|
|||||||
|
|
||||||
mod common;
|
mod common;
|
||||||
|
|
||||||
use crate::common::storage_compat::ecstore::{
|
use crate::common::storage_compat::{
|
||||||
bucket::metadata::BUCKET_LIFECYCLE_CONFIG,
|
BUCKET_LIFECYCLE_CONFIG, BucketVersioningSys, DiskAPI, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
|
||||||
bucket::{
|
GLOBAL_TierConfigMgr, PoolEndpoints, ReadCloser, ReaderImpl, STORAGE_FORMAT_FILE, TierConfig, TierMinIO, TierType,
|
||||||
lifecycle::{TransitionOptions, bucket_lifecycle_ops::enqueue_transition_for_existing_objects},
|
TransitionOptions, WarmBackend, WarmBackendGetOpts, build_transition_put_options, enqueue_transition_for_existing_objects,
|
||||||
metadata_sys,
|
get_bucket_metadata, init_background_expiry, init_bucket_metadata_sys, init_local_disks, new_disk,
|
||||||
versioning_sys::BucketVersioningSys,
|
path2_bucket_object_with_base_path, update_bucket_metadata,
|
||||||
},
|
|
||||||
client::transition_api::{ReadCloser, ReaderImpl},
|
|
||||||
disk::endpoint::Endpoint,
|
|
||||||
disk::{DiskAPI, DiskOption, STORAGE_FORMAT_FILE, new_disk},
|
|
||||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
|
||||||
global::GLOBAL_TierConfigMgr,
|
|
||||||
pools::path2_bucket_object_with_base_path,
|
|
||||||
store::ECStore,
|
|
||||||
tier::{
|
|
||||||
tier_config::{TierConfig, TierMinIO, TierType},
|
|
||||||
warm_backend::{WarmBackend, WarmBackendGetOpts, build_transition_put_options},
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
use futures::FutureExt;
|
use futures::FutureExt;
|
||||||
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
|
use rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT;
|
||||||
@@ -125,9 +113,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
|||||||
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
||||||
|
|
||||||
// format disks (only first time)
|
// format disks (only first time)
|
||||||
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
|
init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
|
// create ECStore with dynamic port 0 (let OS assign) or fixed 9002 if free
|
||||||
let port = 9002; // for simplicity
|
let port = 9002; // for simplicity
|
||||||
@@ -145,11 +131,10 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||||
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||||
|
|
||||||
// Initialize background expiry workers
|
// Initialize background expiry workers
|
||||||
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
|
init_background_expiry(ecstore.clone()).await;
|
||||||
.await;
|
|
||||||
|
|
||||||
// Store in global once lock
|
// Store in global once lock
|
||||||
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
|
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
|
||||||
@@ -197,9 +182,7 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
|
|||||||
};
|
};
|
||||||
|
|
||||||
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
||||||
crate::common::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
|
init_local_disks(endpoint_pools.clone()).await.unwrap();
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
|
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||||
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
|
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
|
||||||
@@ -214,11 +197,10 @@ async fn setup_isolated_test_env(init_expiry: bool) -> (Vec<PathBuf>, Arc<ECStor
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||||
crate::common::storage_compat::ecstore::bucket::metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||||
|
|
||||||
if init_expiry {
|
if init_expiry {
|
||||||
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
|
init_background_expiry(ecstore.clone()).await;
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
(disk_paths, ecstore)
|
(disk_paths, ecstore)
|
||||||
@@ -287,7 +269,7 @@ async fn set_bucket_lifecycle(bucket_name: &str) -> Result<(), Box<dyn std::erro
|
|||||||
</Rule>
|
</Rule>
|
||||||
</LifecycleConfiguration>"#;
|
</LifecycleConfiguration>"#;
|
||||||
|
|
||||||
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
|
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -311,7 +293,7 @@ async fn set_bucket_lifecycle_deletemarker(bucket_name: &str) -> Result<(), Box<
|
|||||||
</Rule>
|
</Rule>
|
||||||
</LifecycleConfiguration>"#;
|
</LifecycleConfiguration>"#;
|
||||||
|
|
||||||
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
|
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec()).await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -334,7 +316,7 @@ async fn set_bucket_lifecycle_delmarker_expiration(bucket_name: &str, days: i64)
|
|||||||
</LifecycleConfiguration>"#
|
</LifecycleConfiguration>"#
|
||||||
);
|
);
|
||||||
|
|
||||||
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
|
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -378,7 +360,7 @@ async fn set_bucket_lifecycle_transition_with_tier(
|
|||||||
</LifecycleConfiguration>"#
|
</LifecycleConfiguration>"#
|
||||||
);
|
);
|
||||||
|
|
||||||
metadata_sys::update(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
|
update_bucket_metadata(bucket_name, BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes()).await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -613,7 +595,7 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
|
|||||||
.await
|
.await
|
||||||
.expect("failed to stat object metadata")
|
.expect("failed to stat object metadata")
|
||||||
.file_type();
|
.file_type();
|
||||||
let lifecycle = metadata_sys::get(bucket)
|
let lifecycle = get_bucket_metadata(bucket)
|
||||||
.await
|
.await
|
||||||
.expect("failed to load bucket metadata")
|
.expect("failed to load bucket metadata")
|
||||||
.lifecycle_config
|
.lifecycle_config
|
||||||
@@ -878,7 +860,7 @@ mod serial_tests {
|
|||||||
println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
|
println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
|
||||||
|
|
||||||
// Verify lifecycle configuration was set
|
// Verify lifecycle configuration was set
|
||||||
match crate::common::storage_compat::ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
|
match get_bucket_metadata(bucket_name.as_str()).await {
|
||||||
Ok(bucket_meta) => {
|
Ok(bucket_meta) => {
|
||||||
assert!(bucket_meta.lifecycle_config.is_some());
|
assert!(bucket_meta.lifecycle_config.is_some());
|
||||||
println!("✅ Bucket metadata retrieved successfully");
|
println!("✅ Bucket metadata retrieved successfully");
|
||||||
@@ -1284,8 +1266,7 @@ mod serial_tests {
|
|||||||
"stale transitioned remote object should still exist before scanner fallback runs"
|
"stale transitioned remote object should still exist before scanner fallback runs"
|
||||||
);
|
);
|
||||||
|
|
||||||
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
|
init_background_expiry(ecstore.clone()).await;
|
||||||
.await;
|
|
||||||
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
@@ -1346,8 +1327,7 @@ mod serial_tests {
|
|||||||
"stale transitioned remote object should still exist before scanner cleanup runs"
|
"stale transitioned remote object should still exist before scanner cleanup runs"
|
||||||
);
|
);
|
||||||
|
|
||||||
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
|
init_background_expiry(ecstore.clone()).await;
|
||||||
.await;
|
|
||||||
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
scan_object_metadata(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
@@ -1437,7 +1417,7 @@ mod serial_tests {
|
|||||||
</Rule>
|
</Rule>
|
||||||
</LifecycleConfiguration>"#
|
</LifecycleConfiguration>"#
|
||||||
);
|
);
|
||||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||||
.await
|
.await
|
||||||
.expect("Failed to set lifecycle configuration");
|
.expect("Failed to set lifecycle configuration");
|
||||||
|
|
||||||
@@ -1522,7 +1502,7 @@ mod serial_tests {
|
|||||||
</Rule>
|
</Rule>
|
||||||
</LifecycleConfiguration>"#
|
</LifecycleConfiguration>"#
|
||||||
);
|
);
|
||||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||||
.await
|
.await
|
||||||
.expect("Failed to set lifecycle configuration");
|
.expect("Failed to set lifecycle configuration");
|
||||||
|
|
||||||
@@ -1714,8 +1694,7 @@ mod serial_tests {
|
|||||||
|
|
||||||
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
|
assert!(object_exists(&ecstore, bucket_name.as_str(), object_name).await);
|
||||||
|
|
||||||
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
|
init_background_expiry(ecstore.clone()).await;
|
||||||
.await;
|
|
||||||
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
@@ -1750,7 +1729,7 @@ mod serial_tests {
|
|||||||
</Rule>
|
</Rule>
|
||||||
</LifecycleConfiguration>"#
|
</LifecycleConfiguration>"#
|
||||||
);
|
);
|
||||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||||
.await
|
.await
|
||||||
.expect("Failed to set lifecycle configuration");
|
.expect("Failed to set lifecycle configuration");
|
||||||
|
|
||||||
@@ -1815,12 +1794,11 @@ mod serial_tests {
|
|||||||
</NoncurrentVersionExpiration>
|
</NoncurrentVersionExpiration>
|
||||||
</Rule>
|
</Rule>
|
||||||
</LifecycleConfiguration>"#;
|
</LifecycleConfiguration>"#;
|
||||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
|
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
|
||||||
.await
|
.await
|
||||||
.expect("Failed to set noncurrent lifecycle configuration");
|
.expect("Failed to set noncurrent lifecycle configuration");
|
||||||
|
|
||||||
crate::common::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone())
|
init_background_expiry(ecstore.clone()).await;
|
||||||
.await;
|
|
||||||
|
|
||||||
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
scan_object_with_lifecycle(&disk_paths[0], bucket_name.as_str(), object_name).await;
|
||||||
|
|
||||||
@@ -1854,7 +1832,7 @@ mod serial_tests {
|
|||||||
</NoncurrentVersionExpiration>
|
</NoncurrentVersionExpiration>
|
||||||
</Rule>
|
</Rule>
|
||||||
</LifecycleConfiguration>"#;
|
</LifecycleConfiguration>"#;
|
||||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
|
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.as_bytes().to_vec())
|
||||||
.await
|
.await
|
||||||
.expect("Failed to set noncurrent lifecycle configuration");
|
.expect("Failed to set noncurrent lifecycle configuration");
|
||||||
|
|
||||||
@@ -1941,7 +1919,7 @@ mod serial_tests {
|
|||||||
</Rule>
|
</Rule>
|
||||||
</LifecycleConfiguration>"#
|
</LifecycleConfiguration>"#
|
||||||
);
|
);
|
||||||
metadata_sys::update(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
update_bucket_metadata(bucket_name.as_str(), BUCKET_LIFECYCLE_CONFIG, lifecycle_xml.into_bytes())
|
||||||
.await
|
.await
|
||||||
.expect("Failed to set lifecycle configuration");
|
.expect("Failed to set lifecycle configuration");
|
||||||
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
|
upload_test_object(&ecstore, bucket_name.as_str(), object_name, b"expire immediately").await;
|
||||||
|
|||||||
@@ -0,0 +1,113 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
use std::fmt;
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum CapabilityState {
|
||||||
|
Supported,
|
||||||
|
Unsupported,
|
||||||
|
Disabled,
|
||||||
|
#[default]
|
||||||
|
Unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CapabilityState {
|
||||||
|
pub const fn is_supported(self) -> bool {
|
||||||
|
matches!(self, Self::Supported)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct CapabilityStatus {
|
||||||
|
pub state: CapabilityState,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub reason: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CapabilityStatus {
|
||||||
|
pub const fn new(state: CapabilityState) -> Self {
|
||||||
|
Self { state, reason: None }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub const fn supported() -> Self {
|
||||||
|
Self::new(CapabilityState::Supported)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub const fn unsupported() -> Self {
|
||||||
|
Self::new(CapabilityState::Unsupported)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub const fn disabled() -> Self {
|
||||||
|
Self::new(CapabilityState::Disabled)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub const fn unknown() -> Self {
|
||||||
|
Self::new(CapabilityState::Unknown)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_reason(mut self, reason: impl Into<String>) -> Self {
|
||||||
|
self.reason = Some(reason.into());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for CapabilityStatus {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::unknown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum CapabilitySnapshotError {
|
||||||
|
Unavailable,
|
||||||
|
Unsupported,
|
||||||
|
InvalidSnapshot(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Display for CapabilitySnapshotError {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
match self {
|
||||||
|
Self::Unavailable => f.write_str("capability snapshot unavailable"),
|
||||||
|
Self::Unsupported => f.write_str("capability snapshot unsupported"),
|
||||||
|
Self::InvalidSnapshot(reason) => write!(f, "invalid capability snapshot: {reason}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::error::Error for CapabilitySnapshotError {}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn capability_status_serializes_unknown_and_unsupported_states() {
|
||||||
|
let unknown = CapabilityStatus::unknown();
|
||||||
|
let unsupported = CapabilityStatus::unsupported().with_reason("target does not expose profiler");
|
||||||
|
|
||||||
|
let encoded = serde_json::to_string(&(unknown, unsupported)).expect("serialize capability statuses");
|
||||||
|
let decoded: (CapabilityStatus, CapabilityStatus) =
|
||||||
|
serde_json::from_str(&encoded).expect("deserialize capability statuses");
|
||||||
|
|
||||||
|
assert_eq!(decoded.0.state, CapabilityState::Unknown);
|
||||||
|
assert_eq!(decoded.1.state, CapabilityState::Unsupported);
|
||||||
|
assert_eq!(decoded.1.reason.as_deref(), Some("target does not expose profiler"));
|
||||||
|
assert!(!decoded.1.state.is_supported());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -16,12 +16,16 @@
|
|||||||
|
|
||||||
pub mod admin;
|
pub mod admin;
|
||||||
pub mod bucket;
|
pub mod bucket;
|
||||||
|
pub mod capability;
|
||||||
pub mod error;
|
pub mod error;
|
||||||
pub mod multipart;
|
pub mod multipart;
|
||||||
pub mod object;
|
pub mod object;
|
||||||
|
pub mod observability;
|
||||||
|
pub mod topology;
|
||||||
|
|
||||||
pub use admin::{DiskSetSelector, StorageAdminApi};
|
pub use admin::{DiskSetSelector, StorageAdminApi};
|
||||||
pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
|
pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
|
||||||
|
pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
|
||||||
pub use error::{StorageErrorCode, StorageResult};
|
pub use error::{StorageErrorCode, StorageResult};
|
||||||
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
|
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
|
||||||
pub use object::{DeletedObject, ObjectToDelete};
|
pub use object::{DeletedObject, ObjectToDelete};
|
||||||
@@ -31,3 +35,10 @@ pub use object::{HealOperations, MultipartOperations, NamespaceLocking, ObjectIO
|
|||||||
pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};
|
pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};
|
||||||
pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};
|
pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};
|
||||||
pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};
|
pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};
|
||||||
|
pub use observability::{
|
||||||
|
MemorySamplingState, ObservabilitySnapshot, ObservabilitySnapshotProvider, PlatformSupport, UserspaceProfilingCapability,
|
||||||
|
};
|
||||||
|
pub use topology::{
|
||||||
|
DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet, TopologySnapshot,
|
||||||
|
TopologySnapshotProvider,
|
||||||
|
};
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
use std::fmt::Debug;
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::{CapabilitySnapshotError, CapabilityStatus};
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct ObservabilitySnapshot {
|
||||||
|
pub runtime_telemetry: CapabilityStatus,
|
||||||
|
pub userspace_profiling: UserspaceProfilingCapability,
|
||||||
|
pub memory_sampling: MemorySamplingState,
|
||||||
|
pub platform: PlatformSupport,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct UserspaceProfilingCapability {
|
||||||
|
pub cpu: CapabilityStatus,
|
||||||
|
pub memory: CapabilityStatus,
|
||||||
|
pub continuous_cpu: CapabilityStatus,
|
||||||
|
pub periodic_cpu: CapabilityStatus,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct MemorySamplingState {
|
||||||
|
pub process: CapabilityStatus,
|
||||||
|
pub system: CapabilityStatus,
|
||||||
|
pub cgroup: CapabilityStatus,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct PlatformSupport {
|
||||||
|
pub target_triple: Option<String>,
|
||||||
|
pub os: Option<String>,
|
||||||
|
pub arch: Option<String>,
|
||||||
|
pub allocator: CapabilityStatus,
|
||||||
|
pub ebpf: CapabilityStatus,
|
||||||
|
pub numa: CapabilityStatus,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
pub trait ObservabilitySnapshotProvider: Send + Sync + Debug {
|
||||||
|
async fn observability_snapshot(&self) -> Result<ObservabilitySnapshot, CapabilitySnapshotError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::CapabilityState;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn observability_snapshot_preserves_unknown_and_unsupported_states() {
|
||||||
|
let snapshot = ObservabilitySnapshot {
|
||||||
|
runtime_telemetry: CapabilityStatus::unknown(),
|
||||||
|
userspace_profiling: UserspaceProfilingCapability {
|
||||||
|
cpu: CapabilityStatus::unsupported().with_reason("unsupported target"),
|
||||||
|
memory: CapabilityStatus::disabled(),
|
||||||
|
continuous_cpu: CapabilityStatus::unknown(),
|
||||||
|
periodic_cpu: CapabilityStatus::supported(),
|
||||||
|
},
|
||||||
|
memory_sampling: MemorySamplingState {
|
||||||
|
process: CapabilityStatus::supported(),
|
||||||
|
system: CapabilityStatus::supported(),
|
||||||
|
cgroup: CapabilityStatus::unknown(),
|
||||||
|
},
|
||||||
|
platform: PlatformSupport {
|
||||||
|
target_triple: Some("x86_64-unknown-linux-gnu".to_owned()),
|
||||||
|
os: Some("linux".to_owned()),
|
||||||
|
arch: Some("x86_64".to_owned()),
|
||||||
|
allocator: CapabilityStatus::supported(),
|
||||||
|
ebpf: CapabilityStatus::unknown(),
|
||||||
|
numa: CapabilityStatus::unsupported(),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
let encoded = serde_json::to_string(&snapshot).expect("serialize observability snapshot");
|
||||||
|
let decoded: ObservabilitySnapshot = serde_json::from_str(&encoded).expect("deserialize observability snapshot");
|
||||||
|
|
||||||
|
assert_eq!(decoded.runtime_telemetry.state, CapabilityState::Unknown);
|
||||||
|
assert_eq!(decoded.userspace_profiling.cpu.state, CapabilityState::Unsupported);
|
||||||
|
assert_eq!(decoded.userspace_profiling.memory.state, CapabilityState::Disabled);
|
||||||
|
assert_eq!(decoded.platform.numa.state, CapabilityState::Unsupported);
|
||||||
|
assert_eq!(decoded.platform.ebpf.state, CapabilityState::Unknown);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::fmt::Debug;
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::{CapabilitySnapshotError, CapabilityStatus};
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct TopologySnapshot {
|
||||||
|
pub pools: Vec<TopologyPool>,
|
||||||
|
pub capabilities: TopologyCapabilities,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct TopologyCapabilities {
|
||||||
|
pub profiling: CapabilityStatus,
|
||||||
|
pub numa: CapabilityStatus,
|
||||||
|
pub failure_domain_labels: CapabilityStatus,
|
||||||
|
pub media_labels: CapabilityStatus,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct TopologyPool {
|
||||||
|
pub pool_index: usize,
|
||||||
|
pub pool_id: Option<String>,
|
||||||
|
pub labels: TopologyLabels,
|
||||||
|
pub sets: Vec<TopologySet>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct TopologySet {
|
||||||
|
pub pool_index: usize,
|
||||||
|
pub set_index: usize,
|
||||||
|
pub set_id: Option<String>,
|
||||||
|
pub labels: TopologyLabels,
|
||||||
|
pub disks: Vec<TopologyDisk>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct TopologyDisk {
|
||||||
|
pub pool_index: usize,
|
||||||
|
pub set_index: usize,
|
||||||
|
pub disk_index: usize,
|
||||||
|
pub disk_id: Option<String>,
|
||||||
|
pub labels: TopologyLabels,
|
||||||
|
pub capabilities: DiskCapabilities,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct TopologyLabels {
|
||||||
|
pub zone: Option<String>,
|
||||||
|
pub rack: Option<String>,
|
||||||
|
pub node: Option<String>,
|
||||||
|
pub media: Option<String>,
|
||||||
|
pub numa_node: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||||
|
pub additional: BTreeMap<String, String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct DiskCapabilities {
|
||||||
|
pub media_type: CapabilityStatus,
|
||||||
|
pub failure_domain: CapabilityStatus,
|
||||||
|
pub numa: CapabilityStatus,
|
||||||
|
pub profiling: CapabilityStatus,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
pub trait TopologySnapshotProvider: Send + Sync + Debug {
|
||||||
|
async fn topology_snapshot(&self) -> Result<TopologySnapshot, CapabilitySnapshotError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::CapabilityState;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn topology_snapshot_allows_missing_and_extra_labels() {
|
||||||
|
let raw = r#"{
|
||||||
|
"pools": [{
|
||||||
|
"pool_index": 0,
|
||||||
|
"pool_id": null,
|
||||||
|
"labels": {
|
||||||
|
"additional": {
|
||||||
|
"room": "a"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"sets": [{
|
||||||
|
"pool_index": 0,
|
||||||
|
"set_index": 1,
|
||||||
|
"set_id": null,
|
||||||
|
"labels": {},
|
||||||
|
"disks": [{
|
||||||
|
"pool_index": 0,
|
||||||
|
"set_index": 1,
|
||||||
|
"disk_index": 2,
|
||||||
|
"disk_id": "disk-2",
|
||||||
|
"labels": {
|
||||||
|
"media": "ssd",
|
||||||
|
"additional": {
|
||||||
|
"slot": "nvme0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"capabilities": {
|
||||||
|
"media_type": { "state": "supported" },
|
||||||
|
"failure_domain": { "state": "unknown" },
|
||||||
|
"numa": { "state": "unsupported", "reason": "not reported" },
|
||||||
|
"profiling": { "state": "disabled" }
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
}]
|
||||||
|
}],
|
||||||
|
"capabilities": {
|
||||||
|
"profiling": { "state": "supported" },
|
||||||
|
"numa": { "state": "unknown" },
|
||||||
|
"failure_domain_labels": { "state": "supported" },
|
||||||
|
"media_labels": { "state": "supported" }
|
||||||
|
}
|
||||||
|
}"#;
|
||||||
|
|
||||||
|
let snapshot: TopologySnapshot = serde_json::from_str(raw).expect("deserialize topology snapshot");
|
||||||
|
let disk = &snapshot.pools[0].sets[0].disks[0];
|
||||||
|
|
||||||
|
assert_eq!(snapshot.pools[0].labels.zone, None);
|
||||||
|
assert_eq!(snapshot.pools[0].labels.additional.get("room").map(String::as_str), Some("a"));
|
||||||
|
assert_eq!(disk.labels.media.as_deref(), Some("ssd"));
|
||||||
|
assert_eq!(disk.labels.additional.get("slot").map(String::as_str), Some("nvme0"));
|
||||||
|
assert_eq!(disk.capabilities.numa.state, CapabilityState::Unsupported);
|
||||||
|
assert_eq!(disk.capabilities.profiling.state, CapabilityState::Disabled);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -92,14 +92,17 @@ Required `rustfs-storage-api` public re-exports:
|
|||||||
|
|
||||||
- `pub use admin::{DiskSetSelector, StorageAdminApi};`
|
- `pub use admin::{DiskSetSelector, StorageAdminApi};`
|
||||||
- `pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};`
|
- `pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};`
|
||||||
|
- `pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};`
|
||||||
- `pub use error::{StorageErrorCode, StorageResult};`
|
- `pub use error::{StorageErrorCode, StorageResult};`
|
||||||
- `pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};`
|
- `pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};`
|
||||||
|
- `pub use observability::{MemorySamplingState, ObservabilitySnapshot, ObservabilitySnapshotProvider, PlatformSupport, UserspaceProfilingCapability};`
|
||||||
- `pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};`
|
- `pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};`
|
||||||
- `pub use object::{ExpirationOptions, TransitionedObject};`
|
- `pub use object::{ExpirationOptions, TransitionedObject};`
|
||||||
- `pub use object::{HealOperations, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations};`
|
- `pub use object::{HealOperations, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations};`
|
||||||
- `pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};`
|
- `pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};`
|
||||||
- `pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};`
|
- `pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};`
|
||||||
- `pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};`
|
- `pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};`
|
||||||
|
- `pub use topology::{DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet, TopologySnapshot, TopologySnapshotProvider};`
|
||||||
|
|
||||||
ECStore must keep compile-time coverage for `StorageAdminApi`, `HealOperations`,
|
ECStore must keep compile-time coverage for `StorageAdminApi`, `HealOperations`,
|
||||||
and the separate `NamespaceLocking` operation group.
|
and the separate `NamespaceLocking` operation group.
|
||||||
|
|||||||
@@ -5,17 +5,19 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
## Current Context
|
## Current Context
|
||||||
|
|
||||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||||
- Branch: `overtrue/arch-storage-api-lifecycle-contracts`
|
- Branch: `overtrue/arch-observability-topology-contracts`
|
||||||
- Baseline: stacked on `rustfs/rustfs#3594` head
|
- Baseline: stacked on `origin/overtrue/arch-test-harness-compat-aliases`
|
||||||
(`6fc05b84e2cd22a0482adfbc042184b03f8fdfa6`).
|
after `rustfs/rustfs#3600`
|
||||||
- PR type for this branch: `pure-move`
|
(`ae6a5befcf60f2f2ebce9799ba93649032234273`).
|
||||||
- Runtime behavior changes: no migration behavior change expected.
|
- PR type for this branch: `contract`
|
||||||
- Rust code changes: move lifecycle helper DTO contracts for expiration and
|
- Runtime behavior changes: none.
|
||||||
transitioned object metadata into rustfs-storage-api, switch ECStore internal
|
- Rust code changes: add observability and topology capability DTO/trait
|
||||||
consumers to direct storage-api imports, and keep ECStore old-path re-exports.
|
contracts to `rustfs-storage-api`.
|
||||||
- CI/script changes: add migration guards rejecting reintroduced ECStore
|
- CI/script changes: extend migration re-export guard coverage for the new
|
||||||
lifecycle helper DTO definitions and old internal consumer imports.
|
contract exports.
|
||||||
- Docs changes: record the lifecycle helper pure-move slice.
|
- Docs changes: add
|
||||||
|
[`runtime-capability-contracts.md`](runtime-capability-contracts.md) and
|
||||||
|
record the combined PR-08/API-013 plus PR-09/API-014 contract slice.
|
||||||
|
|
||||||
## Phase 0 Tasks
|
## Phase 0 Tasks
|
||||||
|
|
||||||
@@ -63,6 +65,60 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
[`ecstore-config-consumer-inventory.md`](ecstore-config-consumer-inventory.md)
|
[`ecstore-config-consumer-inventory.md`](ecstore-config-consumer-inventory.md)
|
||||||
records the current model definitions, global accessors, persistence helpers,
|
records the current model definitions, global accessors, persistence helpers,
|
||||||
consumer groups, migration risks, and do-not-change contract.
|
consumer groups, migration risks, and do-not-change contract.
|
||||||
|
- [x] `G-011` Inventory scheduler baseline.
|
||||||
|
- Acceptance:
|
||||||
|
[`scheduler-baseline.md`](scheduler-baseline.md) records current owners for
|
||||||
|
request admission, reusable scheduler/backpressure facades, workers, scanner
|
||||||
|
budget, heal admission, and the Tokio runtime builder.
|
||||||
|
- Must preserve: no Rust source changes, no scheduler/controller contract
|
||||||
|
changes, and no runtime behavior changes.
|
||||||
|
- [x] `G-012` Inventory placement and repair invariants.
|
||||||
|
- Acceptance:
|
||||||
|
[`placement-repair-invariants.md`](placement-repair-invariants.md) records
|
||||||
|
object-to-set hashing, pool/set/disk assignment boundaries, set-aware
|
||||||
|
readiness and lock quorum, scanner budget, and heal admission preservation
|
||||||
|
gates.
|
||||||
|
- Must preserve: no placement, repair, scanner, heal, readiness, lock, or
|
||||||
|
storage metadata behavior changes.
|
||||||
|
- [x] `G-013` Inventory profiling and NUMA capabilities.
|
||||||
|
- Acceptance:
|
||||||
|
[`profiling-numa-capability-inventory.md`](profiling-numa-capability-inventory.md)
|
||||||
|
records current CPU/memory profiling, cgroup memory sampling, allocator
|
||||||
|
backend, eBPF, and NUMA capability support plus no-op fallback invariants.
|
||||||
|
- Must preserve: no startup, profiling, allocator, runtime, or platform-gate
|
||||||
|
behavior changes.
|
||||||
|
|
||||||
|
## Issue #660 Capability Contract Tasks
|
||||||
|
|
||||||
|
- [x] `PR-08/API-013` Add observability snapshot contract.
|
||||||
|
- Completed slice: add `CapabilityState`, `CapabilityStatus`,
|
||||||
|
`CapabilitySnapshotError`, `ObservabilitySnapshot`,
|
||||||
|
`UserspaceProfilingCapability`, `MemorySamplingState`,
|
||||||
|
`PlatformSupport`, and `ObservabilitySnapshotProvider` to
|
||||||
|
`rustfs-storage-api`.
|
||||||
|
- Acceptance: runtime telemetry, userspace profiling, memory sampling, and
|
||||||
|
platform support states are representable without runtime, ECStore, admin,
|
||||||
|
profiling, exporter, sidecar, eBPF, or OTEL implementation dependencies.
|
||||||
|
- Must preserve: no profiling, startup, admin route, exporter, sidecar, eBPF,
|
||||||
|
OTEL, or runtime behavior changes.
|
||||||
|
- Verification: storage-api contract tests for unknown, unsupported,
|
||||||
|
disabled, and supported capability states; focused storage-api check;
|
||||||
|
migration guard; formatting; diff hygiene; and three-expert review.
|
||||||
|
|
||||||
|
- [x] `PR-09/API-014` Add topology capability contract.
|
||||||
|
- Completed slice: add `TopologySnapshot`, `TopologyCapabilities`,
|
||||||
|
`TopologyPool`, `TopologySet`, `TopologyDisk`, `TopologyLabels`,
|
||||||
|
`DiskCapabilities`, and `TopologySnapshotProvider` to
|
||||||
|
`rustfs-storage-api`.
|
||||||
|
- Acceptance: pool, set, and disk identity fields plus optional zone, rack,
|
||||||
|
node, media, NUMA, and additional labels are representable without
|
||||||
|
`rustfs-ecstore`.
|
||||||
|
- Must preserve: no ECStore endpoint/set implementation, placement,
|
||||||
|
membership, NUMA pinning, or runtime behavior changes.
|
||||||
|
- Verification: storage-api contract tests for missing and additional labels
|
||||||
|
plus supported, unsupported, unknown, and disabled capability states;
|
||||||
|
focused storage-api check; migration guard; formatting; diff hygiene; and
|
||||||
|
three-expert review.
|
||||||
- [x] `TEST-PRTYPE-001` Check PR type enum consistency.
|
- [x] `TEST-PRTYPE-001` Check PR type enum consistency.
|
||||||
- Acceptance: `./scripts/check_architecture_migration_rules.sh` parses the
|
- Acceptance: `./scripts/check_architecture_migration_rules.sh` parses the
|
||||||
allowed PR types from [`crate-boundaries.md`](crate-boundaries.md) and fails
|
allowed PR types from [`crate-boundaries.md`](crate-boundaries.md) and fails
|
||||||
@@ -1296,6 +1352,87 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
migration and layer guards, formatting check, diff hygiene, risk scan, full
|
migration and layer guards, formatting check, diff hygiene, risk scan, full
|
||||||
pre-commit, and required three-expert review passed before push.
|
pre-commit, and required three-expert review passed before push.
|
||||||
|
|
||||||
|
- [x] `API-051` Flatten test harness storage compatibility aliases.
|
||||||
|
- Current branch: `overtrue/arch-test-harness-compat-aliases`.
|
||||||
|
- Current slice: flatten e2e, heal, scanner, and fuzz storage compatibility
|
||||||
|
harnesses from nested `storage_compat::ecstore` modules into direct
|
||||||
|
crate-local aliases, constants, and function imports.
|
||||||
|
- Acceptance: no e2e, heal-test, scanner-test, or fuzz-target harness file
|
||||||
|
may expose or consume nested `storage_compat::ecstore` paths, and migration
|
||||||
|
rules reject reintroducing nested test/fuzz ECStore compatibility modules.
|
||||||
|
- Must preserve: e2e bucket target/RPC/disk helper imports, heal ECStore disk
|
||||||
|
and endpoint setup, scanner lifecycle/tier/disk/storage setup, fuzz bucket
|
||||||
|
validation behavior, and fuzz path-containment validation behavior.
|
||||||
|
- Risk defense: this is test-harness and fuzz-harness import cleanup only; no
|
||||||
|
production runtime behavior, ECStore ownership, storage metadata format, or
|
||||||
|
scanner/heal lifecycle logic is changed.
|
||||||
|
- Verification: focused e2e/heal/scanner test compile, harness tests,
|
||||||
|
migration and layer guards, formatting check, diff hygiene, risk scan, full
|
||||||
|
pre-commit, and required three-expert review passed before push.
|
||||||
|
|
||||||
|
- [x] `API-052` Flatten RustFS runtime storage compatibility aliases.
|
||||||
|
- Current branch: `overtrue/arch-rustfs-storage-compat-aliases`.
|
||||||
|
- Current slice: flatten RustFS root, app, admin, and storage runtime
|
||||||
|
compatibility facades from nested `storage_compat::ecstore` modules into
|
||||||
|
direct crate-local aliases, constants, and function imports.
|
||||||
|
- Acceptance: no RustFS runtime source file may expose or consume nested
|
||||||
|
`storage_compat::ecstore` paths, and migration rules reject reintroducing
|
||||||
|
nested RustFS runtime ECStore compatibility modules.
|
||||||
|
- Must preserve: startup/config/bootstrap behavior, server readiness checks,
|
||||||
|
admin replication/rebalance/tier/config handlers, app object/bucket/
|
||||||
|
multipart usecases, storage RPC/SSE/access paths, table catalog storage
|
||||||
|
access, and existing local compatibility ownership.
|
||||||
|
- Risk defense: this is RustFS runtime import cleanup only; no production
|
||||||
|
runtime behavior, ECStore ownership, storage metadata format, object I/O,
|
||||||
|
admin authorization, or readiness semantics are changed.
|
||||||
|
- Verification: focused RustFS compile, migration and layer guards,
|
||||||
|
formatting check, diff hygiene, risk scan, full pre-commit, and required
|
||||||
|
three-expert review passed before push.
|
||||||
|
|
||||||
|
- [x] `API-053` Flatten RustFS runtime scalar storage compatibility aliases.
|
||||||
|
- Current branch: `overtrue/arch-runtime-compat-surface-prune`.
|
||||||
|
- Current slice: flatten RustFS root, app, admin, and storage runtime scalar
|
||||||
|
compatibility facades such as store, error, global, endpoints, RPC,
|
||||||
|
metrics, notification, set-disk, and data-usage paths into direct
|
||||||
|
crate-local aliases and functions.
|
||||||
|
- Acceptance: RustFS runtime source no longer consumes those scalar
|
||||||
|
compatibility surfaces through secondary modules, while higher-coupling
|
||||||
|
bucket/config/rio compatibility modules remain unchanged; migration rules
|
||||||
|
reject restoring the flattened scalar paths.
|
||||||
|
- Must preserve: startup config/bootstrap behavior, server readiness checks,
|
||||||
|
admin replication/rebalance/tier/config handlers, app object/bucket/
|
||||||
|
multipart usecases, storage RPC/SSE/access paths, table catalog storage
|
||||||
|
access, and existing ECStore concrete type ownership.
|
||||||
|
- Risk defense: this is import ownership and facade-shape cleanup only; no
|
||||||
|
production runtime behavior, ECStore ownership, storage metadata format,
|
||||||
|
object I/O, admin authorization, or readiness semantics are changed.
|
||||||
|
- Verification: focused RustFS compile, migration and layer guards,
|
||||||
|
formatting check, diff hygiene, risk scan, full pre-commit, and required
|
||||||
|
three-expert review passed before push.
|
||||||
|
|
||||||
|
- [x] `API-054` Flatten RustFS runtime secondary storage compatibility aliases.
|
||||||
|
- Current branch: `overtrue/arch-runtime-secondary-compat-flatten`.
|
||||||
|
- Current slice: flatten RustFS root, app, admin, and storage runtime
|
||||||
|
secondary compatibility modules such as bucket, config, rio, client, tier,
|
||||||
|
compress, disk, and rebalance into direct crate-local aliases, modules, and
|
||||||
|
functions.
|
||||||
|
- Acceptance: RustFS runtime source no longer consumes those compatibility
|
||||||
|
surfaces through broad secondary modules, the runtime compatibility files no
|
||||||
|
longer define those wrapper modules, and migration rules reject restoring
|
||||||
|
the flattened secondary paths.
|
||||||
|
- Must preserve: startup config/bootstrap behavior, server module-switch
|
||||||
|
config reads, embedded startup storage initialization, admin bucket/meta/
|
||||||
|
tier/rebalance/config handlers, app object/bucket/multipart usecases,
|
||||||
|
storage RPC/SSE/access paths, table catalog storage access, and ECStore
|
||||||
|
concrete type ownership.
|
||||||
|
- Risk defense: this is import ownership and facade-shape cleanup only; no
|
||||||
|
production runtime behavior, ECStore ownership, storage metadata format,
|
||||||
|
object I/O, admin authorization, tier behavior, or readiness semantics are
|
||||||
|
changed.
|
||||||
|
- Verification: focused RustFS compile, migration and layer guards,
|
||||||
|
formatting check, diff hygiene, risk scan, and required three-expert review
|
||||||
|
passed before push.
|
||||||
|
|
||||||
## Phase 8 Background Controller Tasks
|
## Phase 8 Background Controller Tasks
|
||||||
|
|
||||||
- [x] `BGC-001` Inventory background services.
|
- [x] `BGC-001` Inventory background services.
|
||||||
@@ -1564,7 +1701,10 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
|
|
||||||
## Next PRs
|
## Next PRs
|
||||||
|
|
||||||
1. `pure-move`/`consumer-migration`: continue larger cleanup slices with the
|
1. `contract`/`consumer-migration`: wire read-only observability and topology
|
||||||
|
snapshots to implementation owners without changing runtime, profiling,
|
||||||
|
placement, or admin route behavior.
|
||||||
|
2. `pure-move`/`consumer-migration`: continue larger cleanup slices with the
|
||||||
loss-prevention guards active for remaining ECStore compatibility contracts
|
loss-prevention guards active for remaining ECStore compatibility contracts
|
||||||
now that broad compatibility passthroughs are fully closed.
|
now that broad compatibility passthroughs are fully closed.
|
||||||
|
|
||||||
@@ -1572,14 +1712,65 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
|
|
||||||
| Expert | Status | Notes |
|
| Expert | Status | Notes |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Quality/architecture | passed | S-015 removes obsolete KMS admin policy action variants after the handler fallback cleanup; API-042/API-043/API-044/API-045/API-046/API-047/API-048/API-049/API-050 narrow notify, S3 Select, OBS, IAM, Swift, heal, scanner, RustFS runtime, test, fuzz, and lifecycle helper compatibility contracts without moving ECStore storage metadata ownership. |
|
| Quality/architecture | passed | S-015 removes obsolete KMS admin policy action variants after the handler fallback cleanup; API-042/API-043/API-044/API-045/API-046/API-047/API-048/API-049/API-050/API-051/API-052/API-053/API-054 narrow notify, S3 Select, OBS, IAM, Swift, heal, scanner, RustFS runtime, test, fuzz, lifecycle helper, harness, and RustFS runtime compatibility contracts without moving ECStore storage metadata ownership; G-011/G-012/G-013 add docs-only baselines for scheduler, placement/repair, and profiling/NUMA work; Issue #660 PR-08/PR-09 add read-only observability and topology contracts in rustfs-storage-api only. |
|
||||||
| Migration preservation | passed | KMS endpoint URLs, query aliases, request bodies, response contracts, and dedicated `kms:*` authorization behavior are preserved; event builder call sites, ECStore event bridge conversion, restore event data, version IDs, metadata filtering, config read/save semantics, S3 Select store/error/buffer semantics, OBS metrics state reads, IAM config/notification/error semantics, Swift bucket metadata access, heal disk/resume/task behavior, scanner lifecycle/replication/data-usage behavior, RustFS startup/admin/app/storage runtime access, e2e/test/fuzz import behavior, lifecycle expiration/transition helper DTO field contracts, unchanged no-op handling, and remove-event behavior are preserved. |
|
| Migration preservation | passed | KMS endpoint URLs, query aliases, request bodies, response contracts, and dedicated `kms:*` authorization behavior are preserved; event builder call sites, ECStore event bridge conversion, restore event data, version IDs, metadata filtering, config read/save semantics, S3 Select store/error/buffer semantics, OBS metrics state reads, IAM config/notification/error semantics, Swift bucket metadata access, heal disk/resume/task behavior, scanner lifecycle/replication/data-usage behavior, RustFS startup/admin/app/storage runtime access, e2e/test/fuzz import behavior, lifecycle expiration/transition helper DTO field contracts, flattened harness and RustFS runtime scalar/secondary alias behavior, unchanged no-op handling, remove-event behavior, scheduler/readiness/placement/profiling runtime behavior, platform gates, missing/unknown capability states, and placement/topology labels are preserved. |
|
||||||
| Testing/verification | passed | Focused compiles/tests, guards, formatting, diff hygiene, risk scan, and full `make pre-commit` passed for the current slice. |
|
| Testing/verification | passed | Focused compiles/tests, fuzz target compile, guards, formatting, diff hygiene, risk scan, and full `make pre-commit` passed for prior code slices; current Issue #660 PR-08/PR-09 contract slice uses storage-api tests/checks, migration guard, formatting, diff hygiene, and three-expert review. |
|
||||||
|
|
||||||
## Verification Notes
|
## Verification Notes
|
||||||
|
|
||||||
Passed before push:
|
Passed before push:
|
||||||
|
|
||||||
|
- Issue #660 PR-08/PR-09 current slice:
|
||||||
|
- `cargo test -p rustfs-storage-api`: passed.
|
||||||
|
- `cargo check -p rustfs-storage-api`: passed.
|
||||||
|
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||||
|
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||||
|
- `cargo fmt --all --check`: passed.
|
||||||
|
- `git diff --check`: passed.
|
||||||
|
- `make pre-commit`: passed.
|
||||||
|
- Three-expert review: passed.
|
||||||
|
|
||||||
|
- G-011/G-012/G-013 current slice:
|
||||||
|
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||||
|
- `git diff --check`: passed.
|
||||||
|
- Three-expert review: passed.
|
||||||
|
- Full `make pre-commit`: not run because this slice is documentation-only.
|
||||||
|
|
||||||
|
- API-054 current slice:
|
||||||
|
- `cargo check -p rustfs --lib`: passed.
|
||||||
|
- `cargo check --tests -p rustfs`: passed.
|
||||||
|
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||||
|
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||||
|
- `cargo fmt --all --check`: passed.
|
||||||
|
- `git diff --check`: passed.
|
||||||
|
- Rust risk scan: passed; only existing import and path rewrites were
|
||||||
|
reviewed, with no new unwrap/expect, panic/todo/unsafe, risky casts,
|
||||||
|
ad-hoc error construction, or sensitive-token handling semantics.
|
||||||
|
|
||||||
|
- API-053 current slice:
|
||||||
|
- `cargo check -p rustfs --lib`: passed.
|
||||||
|
- `cargo check --tests -p rustfs`: passed.
|
||||||
|
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||||
|
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||||
|
- `cargo fmt --all --check`: passed.
|
||||||
|
- `git diff --check`: passed.
|
||||||
|
- Rust risk scan: passed; only existing import and path rewrites were
|
||||||
|
reviewed, with no new unwrap/expect, panic/todo/unsafe, risky casts,
|
||||||
|
ad-hoc error construction, or sensitive-token handling semantics.
|
||||||
|
- `make pre-commit`: passed.
|
||||||
|
|
||||||
|
- API-052 current slice:
|
||||||
|
- `cargo check -p rustfs --lib`: passed.
|
||||||
|
- `cargo check --tests -p rustfs`: passed.
|
||||||
|
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||||
|
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||||
|
- `cargo fmt --all --check`: passed.
|
||||||
|
- `git diff --check`: passed.
|
||||||
|
- Rust risk scan: passed; only existing-semantic path replacement hits were
|
||||||
|
reviewed, with no new unwrap/expect, panic/todo/unsafe, risky casts,
|
||||||
|
ad-hoc error construction, or sensitive-token handling semantics.
|
||||||
|
- `make pre-commit`: passed, including 6250 nextest tests and doctests.
|
||||||
|
|
||||||
- API-050 current slice:
|
- API-050 current slice:
|
||||||
- `cargo test -p rustfs-storage-api lifecycle_helper_defaults_preserve_existing_contracts --no-fail-fast`:
|
- `cargo test -p rustfs-storage-api lifecycle_helper_defaults_preserve_existing_contracts --no-fail-fast`:
|
||||||
passed.
|
passed.
|
||||||
@@ -1598,6 +1789,24 @@ Passed before push:
|
|||||||
lines.
|
lines.
|
||||||
- `make pre-commit`: passed.
|
- `make pre-commit`: passed.
|
||||||
|
|
||||||
|
- API-051 current slice:
|
||||||
|
- `cargo check --tests -p e2e_test -p rustfs-heal -p rustfs-scanner`:
|
||||||
|
passed.
|
||||||
|
- `cargo check --manifest-path fuzz/Cargo.toml --all-targets`: passed.
|
||||||
|
- `cargo test -p rustfs-heal --test endpoint_index_test test_endpoint_index_settings --no-fail-fast`:
|
||||||
|
passed.
|
||||||
|
- `cargo test -p rustfs-scanner --test lifecycle_integration_test --no-run`:
|
||||||
|
passed.
|
||||||
|
- `cargo test -p e2e_test --no-run`: passed.
|
||||||
|
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||||
|
- `./scripts/check_layer_dependencies.sh`: passed.
|
||||||
|
- `cargo fmt --all --check`: passed.
|
||||||
|
- `git diff --check`: passed.
|
||||||
|
- Rust risk scan: passed; only existing test `unwrap` calls were touched by
|
||||||
|
import path rewrites, with no new unwrap/expect, panic/todo/unsafe, risky
|
||||||
|
casts, ad-hoc error construction, or sensitive-token handling semantics.
|
||||||
|
- `make pre-commit`: passed.
|
||||||
|
|
||||||
- S-015 current slice:
|
- S-015 current slice:
|
||||||
- `cargo test -p rustfs-policy test_legacy_kms_admin_actions_are_rejected --no-fail-fast`:
|
- `cargo test -p rustfs-policy test_legacy_kms_admin_actions_are_rejected --no-fail-fast`:
|
||||||
passed.
|
passed.
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
# Placement And Repair Invariants
|
||||||
|
|
||||||
|
This inventory covers `G-012` for `rustfs/backlog#666`. It records the current
|
||||||
|
object placement, readiness, lock quorum, scanner, and repair boundaries that
|
||||||
|
later scheduler or topology work must preserve.
|
||||||
|
|
||||||
|
## Object To Set Hash Rule
|
||||||
|
|
||||||
|
Objects reach a set through `Sets::get_disks_by_key`, which calls
|
||||||
|
`get_hashed_set_index` on the object key:
|
||||||
|
|
||||||
|
- `DistributionAlgoVersion::V1` uses `crc_hash(input, set_count)`.
|
||||||
|
- `DistributionAlgoVersion::V2` and `V3` use
|
||||||
|
`sip_hash(input, set_count, format_id_bytes)`.
|
||||||
|
- The format ID is part of the V2/V3 distribution seed, so changing the seed,
|
||||||
|
object key, set count, or algorithm changes placement.
|
||||||
|
|
||||||
|
Preservation rule: every object read, write, list, heal, repair, and
|
||||||
|
decommission path that resolves a set for an existing object must preserve the
|
||||||
|
same object key and format distribution algorithm.
|
||||||
|
|
||||||
|
## Pool, Set, And Disk Assignment Boundary
|
||||||
|
|
||||||
|
Pool selection is separate from set hashing:
|
||||||
|
|
||||||
|
- Existing objects are discovered across pools and resolved to the best current
|
||||||
|
pool candidate before reads or updates continue.
|
||||||
|
- New object writes select an available pool from current per-pool free-space
|
||||||
|
inputs after suspended or rebalancing pools are skipped.
|
||||||
|
- Set selection inside a pool still uses the object-to-set hash rule above.
|
||||||
|
- Disk index assignment comes from endpoint and format metadata, not from a
|
||||||
|
scheduler decision.
|
||||||
|
|
||||||
|
Boundary rule: schedulers may influence admission, worker concurrency, or
|
||||||
|
buffer sizing, but they must not rewrite pool, set, or disk indexes.
|
||||||
|
|
||||||
|
## Readiness And Lock Quorum Boundary
|
||||||
|
|
||||||
|
Runtime readiness currently checks storage and lock health independently:
|
||||||
|
|
||||||
|
- Storage readiness requires every observed set to meet write quorum based on
|
||||||
|
the set drive count and storage class data/parity shape.
|
||||||
|
- Lock readiness aggregates per-set lock-client host quorum and fails fast if
|
||||||
|
any set loses quorum.
|
||||||
|
- Object and bucket mutations acquire namespace locks through the existing
|
||||||
|
storage lock wrappers before changing object or bucket state.
|
||||||
|
|
||||||
|
Boundary rule: readiness and lock quorum must stay set-aware. A global healthy
|
||||||
|
disk count or global connected-host count is not sufficient when any individual
|
||||||
|
set is below quorum.
|
||||||
|
|
||||||
|
## Scanner Budget Preservation
|
||||||
|
|
||||||
|
Scanner cycles are bounded by `ScannerCycleBudget`:
|
||||||
|
|
||||||
|
- Runtime budget cancels the child token after the configured duration.
|
||||||
|
- Object budget cancels after the configured object count.
|
||||||
|
- Directory budget rejects additional directories and cancels with the
|
||||||
|
directories reason.
|
||||||
|
- Partial-cycle metrics and checkpoints use the budget reason.
|
||||||
|
|
||||||
|
Preservation rule: later scheduler work can change how scan cycles are admitted
|
||||||
|
only if it preserves the budget reason, checkpoint reason, and child-token
|
||||||
|
cancellation behavior.
|
||||||
|
|
||||||
|
## Heal Admission Preservation
|
||||||
|
|
||||||
|
Scanner and background repair work enter the heal manager through explicit
|
||||||
|
admission:
|
||||||
|
|
||||||
|
- Scanner object heal requests are low priority and may be accepted, merged,
|
||||||
|
rejected as full, or dropped.
|
||||||
|
- Required/high-priority heal candidates escalate on non-admission instead of
|
||||||
|
silently disappearing.
|
||||||
|
- Heal queue admission deduplicates queued and active work unless the request
|
||||||
|
explicitly forces admission.
|
||||||
|
- Full queues can drop low-priority work or displace lower-priority work for a
|
||||||
|
higher-priority request according to current manager rules.
|
||||||
|
|
||||||
|
Preservation rule: repair scheduling changes must keep admission outcomes
|
||||||
|
observable and must not convert rejected or dropped repair work into silent
|
||||||
|
success.
|
||||||
|
|
||||||
|
## Behavior Change Gates
|
||||||
|
|
||||||
|
Any later placement or repair PR must use the following gates:
|
||||||
|
|
||||||
|
- Placement gate: prove object-to-set hashing is unchanged for existing object
|
||||||
|
keys and format algorithms.
|
||||||
|
- Pool gate: prove pool selection does not choose suspended or rebalancing
|
||||||
|
pools unless the existing path already allows it.
|
||||||
|
- Quorum gate: prove storage readiness and lock readiness remain per-set.
|
||||||
|
- Scanner gate: prove scan budget reason and checkpoint mapping remain stable.
|
||||||
|
- Heal gate: prove low-priority scanner heal, forced heal, duplicate merge, and
|
||||||
|
queue-full outcomes remain distinct.
|
||||||
|
- Rollback gate: if a new scheduler sidecar is disabled, placement and repair
|
||||||
|
must fall back to the current direct ECStore/scanner/heal behavior.
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
# Profiling And NUMA Capability Inventory
|
||||||
|
|
||||||
|
This inventory covers `G-013` for `rustfs/backlog#667`. It records the current
|
||||||
|
profiling, memory sampling, allocator, and NUMA baseline before optional runtime
|
||||||
|
sidecars are designed.
|
||||||
|
|
||||||
|
## Platform Support Matrix
|
||||||
|
|
||||||
|
| Capability | Current support | Current owner | Baseline recommendation |
|
||||||
|
|---|---|---|---|
|
||||||
|
| CPU pprof dump | Linux and macOS builds use `pprof`; other targets return an unsupported-platform error. | `rustfs/src/profiling.rs` | Keep CPU profiling opt-in through existing env flags and cancellation token. |
|
||||||
|
| Continuous CPU profiling | Linux and macOS builds can hold a continuous `ProfilerGuard` when enabled. | `rustfs/src/profiling.rs` | Preserve single-guard ownership and avoid starting multiple continuous guards. |
|
||||||
|
| Periodic CPU profiling | Linux and macOS builds can spawn a periodic sampling loop. | `rustfs/src/profiling.rs` | Keep the loop cancellation-driven and non-fatal. |
|
||||||
|
| Jemalloc memory pprof | Only `linux` + `gnu` + `x86_64` exposes jemalloc pprof dumping. Other supported builds return an unsupported-target error. | `rustfs/src/profiling.rs` | Treat memory pprof as optional and target-gated. |
|
||||||
|
| Periodic memory pprof | Only runs where jemalloc profiling control is available and active. | `rustfs/src/profiling.rs` | Keep inactive jemalloc as a skipped dump, not a startup failure. |
|
||||||
|
| Process/system memory sampling | Uses `rustfs_io_metrics::snapshot_process_resource_and_system` plus `sysinfo` total memory. | `rustfs/src/memory_observability.rs` | Keep sampling portable and metric-gated. |
|
||||||
|
| cgroup memory sampling | Reads Linux cgroup v2 or v1 memory files when present. Missing files produce no cgroup split. | `rustfs/src/memory_observability.rs` | Keep cgroup data opportunistic and absent-safe. |
|
||||||
|
| Allocator reclaim | Uses jemalloc backend on `linux` + `gnu` + `x86_64`; otherwise mimalloc variants. | `rustfs/src/allocator_reclaim.rs` | Keep backend detection read-only and preserve effective-force behavior. |
|
||||||
|
| eBPF | No runtime eBPF sidecar is currently wired into startup. | N/A | Treat eBPF as future optional Linux-only inventory, never as a required baseline. |
|
||||||
|
| NUMA | No NUMA placement or topology controller is currently wired into startup. | N/A | Treat NUMA as future optional capability with no-op fallback. |
|
||||||
|
|
||||||
|
## Cross-Platform Baseline
|
||||||
|
|
||||||
|
The current safe baseline is:
|
||||||
|
|
||||||
|
- Profiling is opt-in through env flags and must not make startup fatal.
|
||||||
|
- Unsupported profiling targets return structured unsupported errors or skip
|
||||||
|
startup tasks.
|
||||||
|
- Memory observability records process/system metrics and adds cgroup split
|
||||||
|
only when cgroup files exist.
|
||||||
|
- Allocator reclaim observes active HTTP, delete-tail, scanner, heal, erasure,
|
||||||
|
and GET-buffer activity before reclaiming.
|
||||||
|
- Runtime thread sizing remains owned by the Tokio runtime builder and sysinfo
|
||||||
|
core detection, not NUMA topology.
|
||||||
|
|
||||||
|
## Optional Sidecar Invariants
|
||||||
|
|
||||||
|
Future sidecars for profiling, eBPF, or NUMA must preserve these invariants:
|
||||||
|
|
||||||
|
- Sidecars must be disabled by default or target-gated until explicitly enabled.
|
||||||
|
- Unsupported targets must degrade to no-op status, not panic or fail startup.
|
||||||
|
- Sidecars must use the runtime cancellation token or an equivalent explicit
|
||||||
|
shutdown handle.
|
||||||
|
- Sidecars must not mutate Tokio worker counts after runtime creation.
|
||||||
|
- Profiling output directory fallback must stay local to profiling and must not
|
||||||
|
affect object storage paths.
|
||||||
|
- NUMA fallback must preserve current runtime thread defaults, storage set
|
||||||
|
placement, and request admission behavior.
|
||||||
|
|
||||||
|
## First Implementation Candidates
|
||||||
|
|
||||||
|
`API-013`:
|
||||||
|
|
||||||
|
- Define a read-only capability contract for profiling, cgroup memory, eBPF,
|
||||||
|
allocator backend, and NUMA availability.
|
||||||
|
- Keep the contract in a low-dependency crate and report unsupported states
|
||||||
|
explicitly.
|
||||||
|
|
||||||
|
`R-016`:
|
||||||
|
|
||||||
|
- Wire storage runtime startup to consume capability snapshots read-only.
|
||||||
|
- Do not start sidecars or mutate runtime worker ownership in the same PR.
|
||||||
|
|
||||||
|
`X-012`:
|
||||||
|
|
||||||
|
- Add an optional profiling/eBPF extension point that can observe capability
|
||||||
|
status.
|
||||||
|
- Keep unsupported targets and disabled sidecars as no-op.
|
||||||
|
|
||||||
|
`X-013`:
|
||||||
|
|
||||||
|
- Add extension tests for disabled, unsupported, and enabled capability
|
||||||
|
snapshots.
|
||||||
|
- Verify no startup fatal boundary is added for optional profiling sidecars.
|
||||||
|
|
||||||
|
`R-017`:
|
||||||
|
|
||||||
|
- If a runtime service sidecar is added later, start it from the startup service
|
||||||
|
boundary with explicit shutdown ownership.
|
||||||
|
- Preserve current service order, KMS/audit/notification fatal boundaries, and
|
||||||
|
scanner/heal startup semantics.
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
# Runtime Capability Contracts
|
||||||
|
|
||||||
|
This document records the `rustfs/backlog#660` PR-08 and PR-09 contract slice.
|
||||||
|
It adds read-only observability and topology snapshot shapes to
|
||||||
|
`rustfs-storage-api` without coupling the contract crate to runtime, ECStore,
|
||||||
|
admin routes, profiling, or observability implementation crates.
|
||||||
|
|
||||||
|
## Observability Snapshot Contract
|
||||||
|
|
||||||
|
`ObservabilitySnapshot` records:
|
||||||
|
|
||||||
|
- Runtime telemetry capability state.
|
||||||
|
- Userspace CPU and memory profiling capability state.
|
||||||
|
- Process, system, and cgroup memory sampling state.
|
||||||
|
- Platform support for target triple, OS, architecture, allocator, eBPF, and
|
||||||
|
NUMA capability.
|
||||||
|
|
||||||
|
Unsupported, disabled, and unknown states are represented by `CapabilityState`
|
||||||
|
instead of failing snapshot construction. The contract is intentionally read
|
||||||
|
only and does not replace existing profiling routes, telemetry APIs, exporter
|
||||||
|
pipelines, or startup behavior.
|
||||||
|
|
||||||
|
## Topology Snapshot Contract
|
||||||
|
|
||||||
|
`TopologySnapshot` records:
|
||||||
|
|
||||||
|
- Pool, set, and disk identity indexes plus optional stable IDs.
|
||||||
|
- Optional zone, rack, node, media, NUMA, and additional labels.
|
||||||
|
- Topology-wide profiling, NUMA, failure-domain label, and media-label
|
||||||
|
capability states.
|
||||||
|
- Per-disk media, failure-domain, NUMA, and profiling capability states.
|
||||||
|
|
||||||
|
Missing labels are represented as absent `Option` values. Extra topology labels
|
||||||
|
belong in the `additional` label map, so future inventory labels do not require
|
||||||
|
ECStore type leakage.
|
||||||
|
|
||||||
|
## Boundary Rules
|
||||||
|
|
||||||
|
- No `rustfs-ecstore`, `rustfs-obs`, Axum, KMS, admin route, OTEL, eBPF, or
|
||||||
|
profiling implementation dependency is added to `rustfs-storage-api`.
|
||||||
|
- No placement, membership, NUMA pinning, profiling, startup, admin route, or
|
||||||
|
exporter behavior changes are part of this contract slice.
|
||||||
|
- Providers must map implementation failures into `CapabilitySnapshotError`
|
||||||
|
before crossing the contract boundary.
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
# Scheduler Baseline Inventory
|
||||||
|
|
||||||
|
This inventory covers `G-011` for `rustfs/backlog#675`. It is a docs-only
|
||||||
|
snapshot of the current scheduling, backpressure, worker, scanner, heal, and
|
||||||
|
runtime-builder ownership. It does not define new behavior.
|
||||||
|
|
||||||
|
## Current Owners
|
||||||
|
|
||||||
|
| Surface | Current owner | Current responsibility | Migration boundary |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `ConcurrencyManager` | `rustfs/src/storage/concurrency/manager.rs` | Owns the RustFS S3 read-path disk-read semaphore, I/O metrics, priority queue, storage media detection, access-pattern detection, and buffer strategy. | Keep request admission and I/O metrics behavior stable until a controller can consume the same state explicitly. |
|
||||||
|
| `SchedulerManager` | `crates/concurrency/src/scheduler.rs` | Provides a reusable facade over `rustfs-io-core::IoScheduler` and derives buffer/priority decisions from `SchedulerPolicy`. | Treat it as reusable library surface; do not assume the RustFS S3 read path has already moved to this facade. |
|
||||||
|
| `BackpressureManager` | `crates/concurrency/src/backpressure.rs` | Provides a reusable duplex-pipe backpressure facade over `rustfs-io-core::BackpressureMonitor`. | Keep pipe sizing and watermark policy separate from object-read disk semaphore admission. |
|
||||||
|
| RustFS backpressure monitor | `rustfs/src/storage/backpressure.rs` | Tracks object-pipe watermark state used by RustFS storage backpressure tests and helpers. | Preserve current state labels and watermark semantics when consolidating with reusable facades. |
|
||||||
|
| `Workers` | `crates/concurrency/src/workers.rs` | Provides cooperative worker-slot admission with `take`, `give`, and `wait`; current background workflows use it for bounded set workers. | Preserve blocking/wakeup semantics and over-release clamping. |
|
||||||
|
| Scanner cycle budget | `crates/scanner/src/scanner_budget.rs` | Cancels a child token when runtime, object-count, or directory-count budget is reached. | Preserve partial-cycle reason mapping and checkpoint accounting. |
|
||||||
|
| Heal admission | `crates/heal/src/heal/manager.rs`, `crates/heal/src/heal/channel.rs`, `rustfs_common::heal_channel` | Owns priority queue admission, duplicate merge/drop/full results, active-task tracking, retry admission, and channel responses. | Preserve low-priority scanner behavior and high-priority escalation gates. |
|
||||||
|
| Tokio runtime builder | `rustfs/src/server/runtime.rs` | Builds the multi-thread runtime from env/defaults, sets thread counts, stack, queue/event intervals, I/O event cap, thread name, and optional dial9 tracing. | Keep runtime defaults and env names stable when later startup phases move ownership. |
|
||||||
|
|
||||||
|
## Current Flow
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
flowchart TD
|
||||||
|
http["HTTP/S3 request"] --> app["app object usecase"]
|
||||||
|
app --> guard["ConcurrencyManager::track_request"]
|
||||||
|
app --> permit["disk-read semaphore permit"]
|
||||||
|
permit --> strategy["I/O queue status and buffer strategy"]
|
||||||
|
strategy --> ecstore["ECStore object/read path"]
|
||||||
|
ecstore --> setdisks["hashed set disks"]
|
||||||
|
|
||||||
|
scanner["scanner cycle"] --> budget["ScannerCycleBudget"]
|
||||||
|
budget --> folder["folder/object scan"]
|
||||||
|
folder --> healreq["heal channel request"]
|
||||||
|
healreq --> admission["HealManager admission queue"]
|
||||||
|
admission --> healworker["heal workers and retries"]
|
||||||
|
|
||||||
|
startup["startup entrypoint"] --> runtime["Tokio runtime builder"]
|
||||||
|
runtime --> services["background services"]
|
||||||
|
```
|
||||||
|
|
||||||
|
## Missing State For Later Work
|
||||||
|
|
||||||
|
`R-015` storage foundation:
|
||||||
|
|
||||||
|
- Needs a stable inventory of endpoint publication, local disk prewarm, lock
|
||||||
|
client setup, and per-set readiness state before any scheduler/controller
|
||||||
|
consumes storage topology.
|
||||||
|
- Must not infer set availability only from request-path I/O metrics.
|
||||||
|
|
||||||
|
`E-011` extension/runtime consumers:
|
||||||
|
|
||||||
|
- Need explicit ownership for runtime admission snapshots before extensions can
|
||||||
|
observe scheduler or backpressure state.
|
||||||
|
- Must not receive mutable handles to `ConcurrencyManager`, heal queues, or
|
||||||
|
scanner budget tokens.
|
||||||
|
|
||||||
|
`C-011` controller work:
|
||||||
|
|
||||||
|
- Needs desired/current/status snapshots for request admission, scanner budget,
|
||||||
|
and heal queue pressure before any controller can reconcile them.
|
||||||
|
- Must keep worker mutation explicit. Read-only status should report `None` or
|
||||||
|
no-op mutation until a reviewed worker lifecycle PR exists.
|
||||||
|
|
||||||
|
## Preservation Invariants
|
||||||
|
|
||||||
|
- Request reads must keep the same disk-read semaphore admission and active GET
|
||||||
|
accounting.
|
||||||
|
- I/O queue status and congestion metrics must remain derived from the same
|
||||||
|
permit counts.
|
||||||
|
- Scanner budget cancellation must keep its reason as runtime, objects, or
|
||||||
|
directories.
|
||||||
|
- Scanner inline-heal compatibility must continue to use asynchronous heal
|
||||||
|
admission.
|
||||||
|
- Heal duplicate admission must prefer merge semantics before full-queue
|
||||||
|
rejection.
|
||||||
|
- High-priority heal admission must still be able to displace lower-priority
|
||||||
|
queued work where the current manager allows it.
|
||||||
|
- Tokio runtime env names and fallback defaults must remain unchanged.
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
mod storage_compat;
|
mod storage_compat;
|
||||||
|
|
||||||
use libfuzzer_sys::fuzz_target;
|
use libfuzzer_sys::fuzz_target;
|
||||||
use crate::storage_compat::ecstore::bucket::utils::{
|
use crate::storage_compat::{
|
||||||
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
|
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -12,12 +12,6 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) use rustfs_ecstore::bucket::utils::{
|
||||||
pub(crate) mod bucket {
|
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
|
||||||
pub(crate) mod utils {
|
};
|
||||||
pub(crate) use rustfs_ecstore::bucket::utils::{
|
|
||||||
check_bucket_and_object_names, check_list_objs_args, check_valid_bucket_name_strict, is_meta_bucketname,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
mod storage_compat;
|
mod storage_compat;
|
||||||
|
|
||||||
use libfuzzer_sys::fuzz_target;
|
use libfuzzer_sys::fuzz_target;
|
||||||
use crate::storage_compat::ecstore::bucket::utils::{check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix};
|
use crate::storage_compat::{check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix};
|
||||||
use rustfs_utils::path::{clean, path_join};
|
use rustfs_utils::path::{clean, path_join};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
|||||||
@@ -12,12 +12,6 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) use rustfs_ecstore::bucket::utils::{
|
||||||
pub(crate) mod bucket {
|
check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix,
|
||||||
pub(crate) mod utils {
|
};
|
||||||
pub(crate) use rustfs_ecstore::bucket::utils::{
|
|
||||||
check_object_name_for_length_and_slash, has_bad_path_component, is_valid_object_prefix,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
use crate::admin::auth::authenticate_request;
|
use crate::admin::auth::authenticate_request;
|
||||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||||
use crate::admin::storage_compat::ecstore::bucket::versioning_sys::BucketVersioningSys;
|
use crate::admin::storage_compat::versioning_sys::BucketVersioningSys;
|
||||||
use crate::app::context::resolve_object_store_handle;
|
use crate::app::context::resolve_object_store_handle;
|
||||||
use crate::auth::get_condition_values;
|
use crate::auth::get_condition_values;
|
||||||
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ pub(crate) async fn load_server_config_from_store() -> S3Result<Config> {
|
|||||||
return Ok(Config::new());
|
return Ok(Config::new());
|
||||||
};
|
};
|
||||||
|
|
||||||
crate::admin::storage_compat::ecstore::config::com::read_config_without_migrate(store)
|
crate::admin::storage_compat::com::read_config_without_migrate(store)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))
|
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))
|
||||||
}
|
}
|
||||||
@@ -82,7 +82,7 @@ where
|
|||||||
return Err(s3_error!(InternalError, "server storage not initialized"));
|
return Err(s3_error!(InternalError, "server storage not initialized"));
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut config = crate::admin::storage_compat::ecstore::config::com::read_config_without_migrate(store.clone())
|
let mut config = crate::admin::storage_compat::com::read_config_without_migrate(store.clone())
|
||||||
.await
|
.await
|
||||||
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
|
.map_err(|e| s3_error!(InternalError, "failed to read server config: {}", e))?;
|
||||||
|
|
||||||
@@ -90,7 +90,7 @@ where
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
crate::admin::storage_compat::ecstore::config::com::save_server_config(store, &config)
|
crate::admin::storage_compat::com::save_server_config(store, &config)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
|
.map_err(|e| s3_error!(InternalError, "failed to save audit config: {}", e))?;
|
||||||
|
|
||||||
|
|||||||
@@ -12,19 +12,17 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::admin::storage_compat::ecstore::bucket::utils::{deserialize, serialize};
|
use crate::admin::storage_compat::utils::{deserialize, serialize};
|
||||||
use crate::admin::storage_compat::ecstore::{
|
use crate::admin::storage_compat::{
|
||||||
bucket::{
|
StorageError,
|
||||||
metadata::{
|
metadata::{
|
||||||
BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
|
BUCKET_LIFECYCLE_CONFIG, BUCKET_NOTIFICATION_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE,
|
||||||
BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG,
|
BUCKET_REPLICATION_CONFIG, BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG,
|
||||||
BucketMetadata, OBJECT_LOCK_CONFIG,
|
BucketMetadata, OBJECT_LOCK_CONFIG,
|
||||||
},
|
|
||||||
metadata_sys,
|
|
||||||
quota::BucketQuota,
|
|
||||||
target::BucketTargets,
|
|
||||||
},
|
},
|
||||||
error::StorageError,
|
metadata_sys,
|
||||||
|
quota::BucketQuota,
|
||||||
|
target::BucketTargets,
|
||||||
};
|
};
|
||||||
use crate::{
|
use crate::{
|
||||||
admin::{
|
admin::{
|
||||||
|
|||||||
@@ -18,12 +18,12 @@ use crate::admin::service::config::{
|
|||||||
apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem, signal_config_snapshot_reload, signal_dynamic_config_reload,
|
apply_dynamic_config_for_subsystem, is_dynamic_config_subsystem, signal_config_snapshot_reload, signal_dynamic_config_reload,
|
||||||
validate_server_config,
|
validate_server_config,
|
||||||
};
|
};
|
||||||
use crate::admin::storage_compat::ecstore::config::com::STORAGE_CLASS_SUB_SYS;
|
use crate::admin::storage_compat::RUSTFS_META_BUCKET;
|
||||||
use crate::admin::storage_compat::ecstore::config::com::{
|
use crate::admin::storage_compat::com::STORAGE_CLASS_SUB_SYS;
|
||||||
|
use crate::admin::storage_compat::com::{
|
||||||
delete_config, read_config, read_config_without_migrate, save_config, save_server_config,
|
delete_config, read_config, read_config_without_migrate, save_config, save_server_config,
|
||||||
};
|
};
|
||||||
use crate::admin::storage_compat::ecstore::config::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
|
use crate::admin::storage_compat::storageclass::{INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV};
|
||||||
use crate::admin::storage_compat::ecstore::disk::RUSTFS_META_BUCKET;
|
|
||||||
use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body};
|
use crate::admin::utils::{encode_compatible_admin_payload, is_compat_admin_request, read_compatible_admin_body};
|
||||||
use crate::app::context::resolve_object_store_handle;
|
use crate::app::context::resolve_object_store_handle;
|
||||||
use crate::auth::{check_key_valid, get_session_token};
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
@@ -711,7 +711,7 @@ fn success_response(config_applied: bool) -> S3Result<S3Response<(StatusCode, Bo
|
|||||||
Ok(S3Response::with_headers((StatusCode::OK, Body::default()), headers))
|
Ok(S3Response::with_headers((StatusCode::OK, Body::default()), headers))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn object_store() -> S3Result<std::sync::Arc<crate::admin::storage_compat::ecstore::store::ECStore>> {
|
fn object_store() -> S3Result<std::sync::Arc<crate::admin::storage_compat::ECStore>> {
|
||||||
resolve_object_store_handle().ok_or_else(|| s3_error!(InternalError, "server storage not initialized"))
|
resolve_object_store_handle().ok_or_else(|| s3_error!(InternalError, "server storage not initialized"))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -756,7 +756,7 @@ fn config_update_sub_system(directives: &[ConfigDirective]) -> S3Result<Option<&
|
|||||||
|
|
||||||
fn validate_config_directives(directives: &[ConfigDirective]) -> S3Result<()> {
|
fn validate_config_directives(directives: &[ConfigDirective]) -> S3Result<()> {
|
||||||
if DEFAULT_KVS.get().is_none() {
|
if DEFAULT_KVS.get().is_none() {
|
||||||
crate::admin::storage_compat::ecstore::config::init();
|
crate::admin::storage_compat::init();
|
||||||
}
|
}
|
||||||
let Some(defaults) = DEFAULT_KVS.get() else {
|
let Some(defaults) = DEFAULT_KVS.get() else {
|
||||||
return Err(s3_error!(InternalError, "config defaults are not initialized"));
|
return Err(s3_error!(InternalError, "config defaults are not initialized"));
|
||||||
@@ -1410,7 +1410,7 @@ fn env_help_key(sub_system: &str, key: &str) -> String {
|
|||||||
|
|
||||||
fn default_help_postfix(sub_system: &str, key: &str) -> String {
|
fn default_help_postfix(sub_system: &str, key: &str) -> String {
|
||||||
if DEFAULT_KVS.get().is_none() {
|
if DEFAULT_KVS.get().is_none() {
|
||||||
crate::admin::storage_compat::ecstore::config::init();
|
crate::admin::storage_compat::init();
|
||||||
}
|
}
|
||||||
|
|
||||||
DEFAULT_KVS
|
DEFAULT_KVS
|
||||||
@@ -1897,7 +1897,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn full_config_export_can_be_reapplied() {
|
fn full_config_export_can_be_reapplied() {
|
||||||
crate::admin::storage_compat::ecstore::config::init();
|
crate::admin::storage_compat::init();
|
||||||
let mut original = ServerConfig::new();
|
let mut original = ServerConfig::new();
|
||||||
apply_set_directives(
|
apply_set_directives(
|
||||||
&mut original,
|
&mut original,
|
||||||
@@ -1944,7 +1944,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn build_help_response_appends_default_value_postfix() {
|
fn build_help_response_appends_default_value_postfix() {
|
||||||
crate::admin::storage_compat::ecstore::config::init();
|
crate::admin::storage_compat::init();
|
||||||
let response = build_help_response(Some("identity_openid"), Some("scopes"), false).expect("help response");
|
let response = build_help_response(Some("identity_openid"), Some("scopes"), false).expect("help response");
|
||||||
|
|
||||||
assert_eq!(response.keys_help.len(), 2);
|
assert_eq!(response.keys_help.len(), 2);
|
||||||
@@ -2057,7 +2057,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn render_selected_config_includes_env_override_lines() {
|
fn render_selected_config_includes_env_override_lines() {
|
||||||
crate::admin::storage_compat::ecstore::config::init();
|
crate::admin::storage_compat::init();
|
||||||
temp_env::with_vars(
|
temp_env::with_vars(
|
||||||
[
|
[
|
||||||
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example")),
|
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example")),
|
||||||
@@ -2093,7 +2093,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn render_selected_config_lists_env_only_targets() {
|
fn render_selected_config_lists_env_only_targets() {
|
||||||
crate::admin::storage_compat::ecstore::config::init();
|
crate::admin::storage_compat::init();
|
||||||
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || {
|
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || {
|
||||||
let config = ServerConfig::new();
|
let config = ServerConfig::new();
|
||||||
let rendered = String::from_utf8(
|
let rendered = String::from_utf8(
|
||||||
@@ -2116,7 +2116,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn render_selected_config_supports_specific_env_only_target_queries() {
|
fn render_selected_config_supports_specific_env_only_target_queries() {
|
||||||
crate::admin::storage_compat::ecstore::config::init();
|
crate::admin::storage_compat::init();
|
||||||
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || {
|
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", Some("http://env.example"))], || {
|
||||||
let config = ServerConfig::new();
|
let config = ServerConfig::new();
|
||||||
let rendered = String::from_utf8(
|
let rendered = String::from_utf8(
|
||||||
@@ -2139,7 +2139,7 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn render_selected_config_orders_default_before_named_targets() {
|
fn render_selected_config_orders_default_before_named_targets() {
|
||||||
crate::admin::storage_compat::ecstore::config::init();
|
crate::admin::storage_compat::init();
|
||||||
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ALPHA", Some("http://alpha.example"))], || {
|
temp_env::with_vars([("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_ALPHA", Some("http://alpha.example"))], || {
|
||||||
let mut config = ServerConfig::new();
|
let mut config = ServerConfig::new();
|
||||||
apply_set_directives(
|
apply_set_directives(
|
||||||
@@ -2320,7 +2320,7 @@ identity_openid client_id="existing-client""#,
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn storage_class_get_target_none_matches_full_export() {
|
fn storage_class_get_target_none_matches_full_export() {
|
||||||
crate::admin::storage_compat::ecstore::config::init();
|
crate::admin::storage_compat::init();
|
||||||
let mut config = ServerConfig::new();
|
let mut config = ServerConfig::new();
|
||||||
apply_set_directives(
|
apply_set_directives(
|
||||||
&mut config,
|
&mut config,
|
||||||
|
|||||||
@@ -14,8 +14,8 @@
|
|||||||
|
|
||||||
use crate::admin::auth::{authenticate_request, validate_admin_request};
|
use crate::admin::auth::{authenticate_request, validate_admin_request};
|
||||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||||
use crate::admin::storage_compat::ecstore::bucket::utils::is_valid_object_prefix;
|
use crate::admin::storage_compat::is_reserved_or_invalid_bucket;
|
||||||
use crate::admin::storage_compat::ecstore::store_utils::is_reserved_or_invalid_bucket;
|
use crate::admin::storage_compat::utils::is_valid_object_prefix;
|
||||||
use crate::app::context::resolve_object_store_handle;
|
use crate::app::context::resolve_object_store_handle;
|
||||||
use crate::server::ADMIN_PREFIX;
|
use crate::server::ADMIN_PREFIX;
|
||||||
use crate::server::RemoteAddr;
|
use crate::server::RemoteAddr;
|
||||||
@@ -345,10 +345,10 @@ fn should_handle_root_heal_directly(hip: &HealInitParams) -> bool {
|
|||||||
hip.bucket.is_empty() && hip.obj_prefix.is_empty() && hip.client_token.is_empty() && !hip.force_stop
|
hip.bucket.is_empty() && hip.obj_prefix.is_empty() && hip.client_token.is_empty() && !hip.force_stop
|
||||||
}
|
}
|
||||||
|
|
||||||
fn map_root_heal_status(heal_err: Option<crate::admin::storage_compat::ecstore::error::Error>) -> S3Result<()> {
|
fn map_root_heal_status(heal_err: Option<crate::admin::storage_compat::Error>) -> S3Result<()> {
|
||||||
match heal_err {
|
match heal_err {
|
||||||
None => Ok(()),
|
None => Ok(()),
|
||||||
Some(crate::admin::storage_compat::ecstore::error::StorageError::NoHealRequired) => {
|
Some(crate::admin::storage_compat::StorageError::NoHealRequired) => {
|
||||||
info!(
|
info!(
|
||||||
event = EVENT_ADMIN_RESPONSE_EMITTED,
|
event = EVENT_ADMIN_RESPONSE_EMITTED,
|
||||||
component = LOG_COMPONENT_ADMIN_API,
|
component = LOG_COMPONENT_ADMIN_API,
|
||||||
@@ -703,7 +703,7 @@ mod tests {
|
|||||||
encode_heal_task_status, heal_channel_response_items, heal_channel_response_summary, json_response, map_heal_response,
|
encode_heal_task_status, heal_channel_response_items, heal_channel_response_summary, json_response, map_heal_response,
|
||||||
map_root_heal_status, should_handle_root_heal_directly, validate_heal_request_mode, validate_heal_target,
|
map_root_heal_status, should_handle_root_heal_directly, validate_heal_request_mode, validate_heal_target,
|
||||||
};
|
};
|
||||||
use crate::admin::storage_compat::ecstore::error::StorageError;
|
use crate::admin::storage_compat::StorageError;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use http::StatusCode;
|
use http::StatusCode;
|
||||||
use http::Uri;
|
use http::Uri;
|
||||||
|
|||||||
@@ -16,7 +16,7 @@
|
|||||||
|
|
||||||
use crate::admin::auth::validate_admin_request;
|
use crate::admin::auth::validate_admin_request;
|
||||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||||
use crate::admin::storage_compat::ecstore::config::com::{read_config, save_config};
|
use crate::admin::storage_compat::com::{read_config, save_config};
|
||||||
use crate::app::context::{resolve_kms_runtime_service_manager, resolve_object_store_handle};
|
use crate::app::context::{resolve_kms_runtime_service_manager, resolve_object_store_handle};
|
||||||
use crate::auth::{check_key_valid, get_session_token};
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||||
|
|||||||
@@ -20,7 +20,7 @@
|
|||||||
|
|
||||||
use crate::admin::auth::validate_admin_request;
|
use crate::admin::auth::validate_admin_request;
|
||||||
use crate::admin::router::Operation;
|
use crate::admin::router::Operation;
|
||||||
use crate::admin::storage_compat::ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
use crate::admin::storage_compat::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
||||||
use crate::auth::{check_key_valid, get_session_token};
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
use crate::server::RemoteAddr;
|
use crate::server::RemoteAddr;
|
||||||
use crate::storage::request_context::spawn_traced;
|
use crate::storage::request_context::spawn_traced;
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, Operation, S3Router};
|
use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, Operation, S3Router};
|
||||||
use crate::admin::storage_compat::ecstore::global::get_global_region;
|
use crate::admin::storage_compat::get_global_region;
|
||||||
use crate::app::context::resolve_object_store_handle;
|
use crate::app::context::resolve_object_store_handle;
|
||||||
use crate::auth::{check_key_valid, get_session_token};
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
use crate::error::ApiError;
|
use crate::error::ApiError;
|
||||||
@@ -649,7 +649,7 @@ async fn preflight_zip_items(request: &CreateObjectZipDownloadRequest, items: &[
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn storage_error_to_s3(err: crate::admin::storage_compat::ecstore::error::Error) -> s3s::S3Error {
|
fn storage_error_to_s3(err: crate::admin::storage_compat::Error) -> s3s::S3Error {
|
||||||
ApiError::from(err).into()
|
ApiError::from(err).into()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@
|
|||||||
use super::sts::create_oidc_sts_credentials;
|
use super::sts::create_oidc_sts_credentials;
|
||||||
use crate::admin::auth::validate_admin_request;
|
use crate::admin::auth::validate_admin_request;
|
||||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||||
use crate::admin::storage_compat::ecstore::config::com::{read_config_without_migrate, save_server_config};
|
use crate::admin::storage_compat::com::{read_config_without_migrate, save_server_config};
|
||||||
use crate::app::context::resolve_object_store_handle;
|
use crate::app::context::resolve_object_store_handle;
|
||||||
use crate::auth::{check_key_valid, get_session_token};
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
use crate::server::{ADMIN_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr};
|
use crate::server::{ADMIN_PREFIX, MINIO_ADMIN_PREFIX, RemoteAddr};
|
||||||
|
|||||||
@@ -99,7 +99,7 @@ macro_rules! log_pool_response_emitted {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
fn endpoints_from_context() -> Option<crate::admin::storage_compat::ecstore::endpoints::EndpointServerPools> {
|
fn endpoints_from_context() -> Option<crate::admin::storage_compat::EndpointServerPools> {
|
||||||
resolve_endpoints_handle()
|
resolve_endpoints_handle()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,9 +16,9 @@
|
|||||||
|
|
||||||
use crate::admin::auth::{validate_admin_request, validate_admin_request_with_bucket};
|
use crate::admin::auth::{validate_admin_request, validate_admin_request_with_bucket};
|
||||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||||
use crate::admin::storage_compat::ecstore::bucket::metadata_sys::BucketMetadataSys;
|
use crate::admin::storage_compat::metadata_sys::BucketMetadataSys;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::quota::checker::QuotaChecker;
|
use crate::admin::storage_compat::quota::checker::QuotaChecker;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::quota::{BucketQuota, QuotaError, QuotaOperation};
|
use crate::admin::storage_compat::quota::{BucketQuota, QuotaError, QuotaOperation};
|
||||||
use crate::app::context::{resolve_bucket_metadata_handle, resolve_object_store_handle};
|
use crate::app::context::{resolve_bucket_metadata_handle, resolve_object_store_handle};
|
||||||
use crate::auth::{check_key_valid, get_session_token};
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
use crate::server::ADMIN_PREFIX;
|
use crate::server::ADMIN_PREFIX;
|
||||||
@@ -175,7 +175,7 @@ async fn current_usage_from_context(bucket: &str) -> u64 {
|
|||||||
return 0;
|
return 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
match crate::admin::storage_compat::ecstore::data_usage::load_data_usage_from_backend(store).await {
|
match crate::admin::storage_compat::load_data_usage_from_backend(store).await {
|
||||||
Ok(data_usage_info) => data_usage_info
|
Ok(data_usage_info) => data_usage_info
|
||||||
.buckets_usage
|
.buckets_usage
|
||||||
.get(bucket)
|
.get(bucket)
|
||||||
|
|||||||
@@ -12,10 +12,8 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::admin::storage_compat::ecstore::{
|
use crate::admin::storage_compat::{
|
||||||
error::StorageError,
|
DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, StorageError, get_global_notification_sys,
|
||||||
notification_sys::get_global_notification_sys,
|
|
||||||
rebalance::{DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta},
|
|
||||||
};
|
};
|
||||||
use crate::{
|
use crate::{
|
||||||
admin::{
|
admin::{
|
||||||
@@ -150,7 +148,7 @@ fn build_rebalance_pool_progress(
|
|||||||
now: OffsetDateTime,
|
now: OffsetDateTime,
|
||||||
stop_time: Option<OffsetDateTime>,
|
stop_time: Option<OffsetDateTime>,
|
||||||
percent_free_goal: f64,
|
percent_free_goal: f64,
|
||||||
ps: &crate::admin::storage_compat::ecstore::rebalance::RebalanceStats,
|
ps: &crate::admin::storage_compat::RebalanceStats,
|
||||||
) -> Option<RebalPoolProgress> {
|
) -> Option<RebalPoolProgress> {
|
||||||
let total_bytes_to_rebal = ps.init_capacity as f64 * percent_free_goal - ps.init_free_space as f64;
|
let total_bytes_to_rebal = ps.init_capacity as f64 * percent_free_goal - ps.init_free_space as f64;
|
||||||
let terminal_time = ps.info.end_time.or(stop_time);
|
let terminal_time = ps.info.end_time.or(stop_time);
|
||||||
@@ -193,7 +191,7 @@ fn build_rebalance_pool_statuses(
|
|||||||
now: OffsetDateTime,
|
now: OffsetDateTime,
|
||||||
stop_time: Option<OffsetDateTime>,
|
stop_time: Option<OffsetDateTime>,
|
||||||
percent_free_goal: f64,
|
percent_free_goal: f64,
|
||||||
pool_stats: &[crate::admin::storage_compat::ecstore::rebalance::RebalanceStats],
|
pool_stats: &[crate::admin::storage_compat::RebalanceStats],
|
||||||
disk_stats: &[DiskStat],
|
disk_stats: &[DiskStat],
|
||||||
) -> Vec<RebalancePoolStatus> {
|
) -> Vec<RebalancePoolStatus> {
|
||||||
pool_stats
|
pool_stats
|
||||||
@@ -563,9 +561,7 @@ mod rebalance_handler_tests {
|
|||||||
RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, build_rebalance_pool_statuses, rebalance_pool_used,
|
RebalPoolProgress, RebalanceAdminStatus, RebalancePoolStatus, build_rebalance_pool_statuses, rebalance_pool_used,
|
||||||
rebalance_remaining_buckets, rebalance_used_pct,
|
rebalance_remaining_buckets, rebalance_used_pct,
|
||||||
};
|
};
|
||||||
use crate::admin::storage_compat::ecstore::rebalance::{
|
use crate::admin::storage_compat::{DiskStat, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceStats};
|
||||||
DiskStat, RebalStatus, RebalanceCleanupWarnings, RebalanceInfo, RebalanceStats,
|
|
||||||
};
|
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -15,15 +15,15 @@
|
|||||||
use crate::admin::auth::validate_admin_request;
|
use crate::admin::auth::validate_admin_request;
|
||||||
use crate::admin::handlers::site_replication::site_replication_peer_deployment_id_for_endpoint;
|
use crate::admin::handlers::site_replication::site_replication_peer_deployment_id_for_endpoint;
|
||||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||||
use crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::{BucketTargetError, BucketTargetSys};
|
use crate::admin::storage_compat::StorageError;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::metadata::BUCKET_TARGETS_FILE;
|
use crate::admin::storage_compat::bucket_target_sys::{BucketTargetError, BucketTargetSys};
|
||||||
use crate::admin::storage_compat::ecstore::bucket::metadata_sys;
|
use crate::admin::storage_compat::global_rustfs_port;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::metadata_sys::get_replication_config;
|
use crate::admin::storage_compat::metadata::BUCKET_TARGETS_FILE;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::replication::BucketStats;
|
use crate::admin::storage_compat::metadata_sys;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
|
use crate::admin::storage_compat::metadata_sys::get_replication_config;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::target::BucketTarget;
|
use crate::admin::storage_compat::replication::BucketStats;
|
||||||
use crate::admin::storage_compat::ecstore::error::StorageError;
|
use crate::admin::storage_compat::replication::GLOBAL_REPLICATION_STATS;
|
||||||
use crate::admin::storage_compat::ecstore::global::global_rustfs_port;
|
use crate::admin::storage_compat::target::BucketTarget;
|
||||||
use crate::admin::utils::read_compatible_admin_body;
|
use crate::admin::utils::read_compatible_admin_body;
|
||||||
use crate::app::context::resolve_object_store_handle;
|
use crate::app::context::resolve_object_store_handle;
|
||||||
use crate::auth::{check_key_valid, get_session_token};
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
|
|||||||
@@ -18,24 +18,20 @@ use crate::admin::site_replication_identity::{
|
|||||||
canonical_endpoint, deployment_id_for_endpoint, normalize_peer_map_by_identity_with, same_identity_endpoint,
|
canonical_endpoint, deployment_id_for_endpoint, normalize_peer_map_by_identity_with, same_identity_endpoint,
|
||||||
site_identity_key,
|
site_identity_key,
|
||||||
};
|
};
|
||||||
use crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::BucketTargetSys;
|
use crate::admin::storage_compat::Error as StorageError;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::metadata::{
|
use crate::admin::storage_compat::bucket_target_sys::BucketTargetSys;
|
||||||
|
use crate::admin::storage_compat::com::{delete_config, read_config, save_config};
|
||||||
|
use crate::admin::storage_compat::metadata::{
|
||||||
BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG,
|
BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_POLICY_CONFIG, BUCKET_QUOTA_CONFIG_FILE, BUCKET_REPLICATION_CONFIG,
|
||||||
BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG,
|
BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_TARGETS_FILE, BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG,
|
||||||
};
|
};
|
||||||
use crate::admin::storage_compat::ecstore::bucket::metadata_sys;
|
use crate::admin::storage_compat::metadata_sys;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::replication::GLOBAL_REPLICATION_STATS;
|
use crate::admin::storage_compat::replication::GLOBAL_REPLICATION_STATS;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::replication::{
|
use crate::admin::storage_compat::replication::{ReplicationConfigurationExt, ResyncOpts, get_global_replication_pool};
|
||||||
ReplicationConfigurationExt, ResyncOpts, get_global_replication_pool,
|
use crate::admin::storage_compat::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials};
|
||||||
};
|
use crate::admin::storage_compat::utils::{deserialize, serialize};
|
||||||
use crate::admin::storage_compat::ecstore::bucket::target::{ARN, BucketTarget, BucketTargetType, BucketTargets, Credentials};
|
use crate::admin::storage_compat::versioning::VersioningApi;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::utils::{deserialize, serialize};
|
use crate::admin::storage_compat::{get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port};
|
||||||
use crate::admin::storage_compat::ecstore::bucket::versioning::VersioningApi;
|
|
||||||
use crate::admin::storage_compat::ecstore::config::com::{delete_config, read_config, save_config};
|
|
||||||
use crate::admin::storage_compat::ecstore::error::Error as StorageError;
|
|
||||||
use crate::admin::storage_compat::ecstore::global::{
|
|
||||||
get_global_deployment_id, get_global_endpoints_opt, get_global_region, global_rustfs_port,
|
|
||||||
};
|
|
||||||
use crate::admin::utils::{encode_compatible_admin_payload, read_compatible_admin_body};
|
use crate::admin::utils::{encode_compatible_admin_payload, read_compatible_admin_body};
|
||||||
use crate::app::context::resolve_object_store_handle;
|
use crate::app::context::resolve_object_store_handle;
|
||||||
use crate::auth::{check_key_valid, get_session_token};
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
@@ -656,7 +652,7 @@ async fn site_replication_peer_client() -> S3Result<reqwest::Client> {
|
|||||||
built
|
built
|
||||||
}
|
}
|
||||||
|
|
||||||
fn runtime_tls_enabled_with(endpoints: Option<&crate::admin::storage_compat::ecstore::endpoints::EndpointServerPools>) -> bool {
|
fn runtime_tls_enabled_with(endpoints: Option<&crate::admin::storage_compat::EndpointServerPools>) -> bool {
|
||||||
if !rustfs_utils::get_env_str(ENV_RUSTFS_TLS_PATH, DEFAULT_RUSTFS_TLS_PATH).is_empty() {
|
if !rustfs_utils::get_env_str(ENV_RUSTFS_TLS_PATH, DEFAULT_RUSTFS_TLS_PATH).is_empty() {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -3357,7 +3353,7 @@ fn is_stale_update(local_updated_at: OffsetDateTime, incoming_updated_at: Option
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn bucket_meta_local_updated_at(
|
fn bucket_meta_local_updated_at(
|
||||||
bucket_meta: &crate::admin::storage_compat::ecstore::bucket::metadata::BucketMetadata,
|
bucket_meta: &crate::admin::storage_compat::metadata::BucketMetadata,
|
||||||
config_file: &str,
|
config_file: &str,
|
||||||
) -> OffsetDateTime {
|
) -> OffsetDateTime {
|
||||||
match config_file {
|
match config_file {
|
||||||
@@ -4578,8 +4574,8 @@ impl Operation for SRRotateServiceAccountHandler {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::admin::storage_compat::ecstore::disk::endpoint::Endpoint;
|
use crate::admin::storage_compat::Endpoint;
|
||||||
use crate::admin::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
use crate::admin::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||||
use http::{HeaderMap, HeaderValue, Uri};
|
use http::{HeaderMap, HeaderValue, Uri};
|
||||||
use rustfs_common::{get_global_outbound_tls_generation, set_global_outbound_tls_generation};
|
use rustfs_common::{get_global_outbound_tls_generation, set_global_outbound_tls_generation};
|
||||||
use rustfs_policy::policy::action::S3Action;
|
use rustfs_policy::policy::action::S3Action;
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use super::is_admin::IsAdminHandler;
|
use super::is_admin::IsAdminHandler;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::utils::serialize;
|
use crate::admin::storage_compat::utils::serialize;
|
||||||
use crate::{
|
use crate::{
|
||||||
admin::{
|
admin::{
|
||||||
handlers::site_replication::site_replication_iam_change_hook,
|
handlers::site_replication::site_replication_iam_change_hook,
|
||||||
|
|||||||
@@ -12,10 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::admin::storage_compat::ecstore::{
|
use crate::admin::storage_compat::{ECStore, metadata::table_catalog_path_hash, metadata_sys};
|
||||||
bucket::{metadata::table_catalog_path_hash, metadata_sys},
|
|
||||||
store::ECStore,
|
|
||||||
};
|
|
||||||
use crate::admin::{
|
use crate::admin::{
|
||||||
auth::{AdminResourceScope, validate_admin_request, validate_admin_request_with_bucket_object},
|
auth::{AdminResourceScope, validate_admin_request, validate_admin_request_with_bucket_object},
|
||||||
router::{AdminOperation, Operation, S3Router},
|
router::{AdminOperation, Operation, S3Router},
|
||||||
|
|||||||
@@ -13,21 +13,11 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
#![allow(unused_variables, unused_mut, unused_must_use)]
|
#![allow(unused_variables, unused_mut, unused_must_use)]
|
||||||
|
|
||||||
use crate::admin::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState;
|
use crate::admin::storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_TransitionState;
|
||||||
use crate::admin::storage_compat::ecstore::{
|
use crate::admin::storage_compat::{
|
||||||
bucket::lifecycle::tier_last_day_stats::DailyAllTierStats,
|
AdminError, DailyAllTierStats, ERR_TIER_ALREADY_EXISTS, ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY,
|
||||||
client::admin_handler_utils::AdminError,
|
ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_MISSING_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE,
|
||||||
config::storageclass,
|
ERR_TIER_NOT_FOUND, TierConfig, TierCreds, TierType, get_global_notification_sys, storageclass,
|
||||||
notification_sys::get_global_notification_sys,
|
|
||||||
tier::{
|
|
||||||
tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_MISSING_CREDENTIALS},
|
|
||||||
tier_admin::TierCreds,
|
|
||||||
tier_config::{TierConfig, TierType},
|
|
||||||
tier_handlers::{
|
|
||||||
ERR_TIER_ALREADY_EXISTS, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE,
|
|
||||||
ERR_TIER_NOT_FOUND,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
use crate::{
|
use crate::{
|
||||||
admin::{
|
admin::{
|
||||||
@@ -916,7 +906,7 @@ impl Operation for ClearTier {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::lifecycle::tier_last_day_stats::LastDayTierStats;
|
use crate::admin::storage_compat::lifecycle::tier_last_day_stats::LastDayTierStats;
|
||||||
use http::Uri;
|
use http::Uri;
|
||||||
use matchit::Router;
|
use matchit::Router;
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::admin::router::Operation;
|
use crate::admin::router::Operation;
|
||||||
use crate::admin::storage_compat::ecstore::rpc::PeerRestClient;
|
use crate::admin::storage_compat::PeerRestClient;
|
||||||
use crate::app::context::resolve_endpoints_handle;
|
use crate::app::context::resolve_endpoints_handle;
|
||||||
use http::StatusCode;
|
use http::StatusCode;
|
||||||
use hyper::Uri;
|
use hyper::Uri;
|
||||||
|
|||||||
+25
-27
@@ -14,24 +14,24 @@
|
|||||||
|
|
||||||
use crate::admin::console::{is_console_path, make_console_server};
|
use crate::admin::console::{is_console_path, make_console_server};
|
||||||
use crate::admin::handlers::oidc::is_oidc_path;
|
use crate::admin::handlers::oidc::is_oidc_path;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::bandwidth::monitor::BandwidthDetails;
|
use crate::admin::storage_compat::GLOBAL_BOOT_TIME;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::{
|
use crate::admin::storage_compat::PeerRestClient;
|
||||||
|
use crate::admin::storage_compat::bandwidth::monitor::BandwidthDetails;
|
||||||
|
use crate::admin::storage_compat::bucket_target_sys::{
|
||||||
BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, TargetClient,
|
BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, TargetClient,
|
||||||
};
|
};
|
||||||
use crate::admin::storage_compat::ecstore::bucket::metadata::BUCKET_TARGETS_FILE;
|
use crate::admin::storage_compat::com::read_config_without_migrate;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::metadata_sys;
|
use crate::admin::storage_compat::get_global_notification_sys;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::replication::{
|
use crate::admin::storage_compat::metadata::BUCKET_TARGETS_FILE;
|
||||||
|
use crate::admin::storage_compat::metadata_sys;
|
||||||
|
use crate::admin::storage_compat::replication::{
|
||||||
BucketReplicationResyncStatus, BucketStats, GLOBAL_REPLICATION_STATS, ObjectOpts, ReplicationConfigurationExt, ResyncOpts,
|
BucketReplicationResyncStatus, BucketStats, GLOBAL_REPLICATION_STATS, ObjectOpts, ReplicationConfigurationExt, ResyncOpts,
|
||||||
get_global_replication_pool,
|
get_global_replication_pool,
|
||||||
};
|
};
|
||||||
use crate::admin::storage_compat::ecstore::bucket::target::{BucketTarget, BucketTargetType, BucketTargets};
|
use crate::admin::storage_compat::target::{BucketTarget, BucketTargetType, BucketTargets};
|
||||||
use crate::admin::storage_compat::ecstore::bucket::versioning::VersioningApi;
|
use crate::admin::storage_compat::versioning::VersioningApi;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::versioning_sys::BucketVersioningSys;
|
use crate::admin::storage_compat::versioning_sys::BucketVersioningSys;
|
||||||
use crate::admin::storage_compat::ecstore::config::com::read_config_without_migrate;
|
use crate::admin::storage_compat::{get_global_bucket_monitor, get_global_deployment_id, get_global_region};
|
||||||
use crate::admin::storage_compat::ecstore::global::GLOBAL_BOOT_TIME;
|
|
||||||
use crate::admin::storage_compat::ecstore::global::{get_global_bucket_monitor, get_global_deployment_id, get_global_region};
|
|
||||||
use crate::admin::storage_compat::ecstore::notification_sys::get_global_notification_sys;
|
|
||||||
use crate::admin::storage_compat::ecstore::rpc::PeerRestClient;
|
|
||||||
use crate::app::context::resolve_object_store_handle;
|
use crate::app::context::resolve_object_store_handle;
|
||||||
use crate::app::object_usecase::DefaultObjectUsecase;
|
use crate::app::object_usecase::DefaultObjectUsecase;
|
||||||
use crate::auth::{check_key_valid, get_session_token};
|
use crate::auth::{check_key_valid, get_session_token};
|
||||||
@@ -1420,9 +1420,7 @@ async fn ensure_replication_bucket_exists(bucket: &str) -> S3Result<()> {
|
|||||||
async fn ensure_replication_config_exists(bucket: &str) -> S3Result<()> {
|
async fn ensure_replication_config_exists(bucket: &str) -> S3Result<()> {
|
||||||
match metadata_sys::get_replication_config(bucket).await {
|
match metadata_sys::get_replication_config(bucket).await {
|
||||||
Ok(_) => Ok(()),
|
Ok(_) => Ok(()),
|
||||||
Err(crate::admin::storage_compat::ecstore::error::StorageError::ConfigNotFound) => {
|
Err(crate::admin::storage_compat::StorageError::ConfigNotFound) => Err(s3_error!(ReplicationConfigurationNotFoundError)),
|
||||||
Err(s3_error!(ReplicationConfigurationNotFoundError))
|
|
||||||
}
|
|
||||||
Err(err) => Err(ApiError::from(err).into()),
|
Err(err) => Err(ApiError::from(err).into()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1947,7 +1945,7 @@ async fn resolve_replication_target_client(bucket: &str, target: &BucketTarget)
|
|||||||
|
|
||||||
fn build_replication_probe_put_options(now: OffsetDateTime) -> PutObjectOptions {
|
fn build_replication_probe_put_options(now: OffsetDateTime) -> PutObjectOptions {
|
||||||
PutObjectOptions {
|
PutObjectOptions {
|
||||||
internal: crate::admin::storage_compat::ecstore::bucket::bucket_target_sys::AdvancedPutOptions {
|
internal: crate::admin::storage_compat::bucket_target_sys::AdvancedPutOptions {
|
||||||
source_version_id: Uuid::new_v4().to_string(),
|
source_version_id: Uuid::new_v4().to_string(),
|
||||||
replication_status: ReplicationStatusType::Replica,
|
replication_status: ReplicationStatusType::Replica,
|
||||||
source_mtime: now,
|
source_mtime: now,
|
||||||
@@ -2062,7 +2060,7 @@ async fn source_bucket_requires_object_lock(bucket: &str) -> S3Result<bool> {
|
|||||||
.object_lock_enabled
|
.object_lock_enabled
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.is_some_and(|state| state.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)),
|
.is_some_and(|state| state.as_str() == s3s::dto::ObjectLockEnabled::ENABLED)),
|
||||||
Err(crate::admin::storage_compat::ecstore::error::StorageError::ConfigNotFound) => Ok(false),
|
Err(crate::admin::storage_compat::StorageError::ConfigNotFound) => Ok(false),
|
||||||
Err(err) => Err(ApiError::from(err).into()),
|
Err(err) => Err(ApiError::from(err).into()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2776,7 +2774,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn apply_replication_reset_to_targets_updates_matching_target() {
|
fn apply_replication_reset_to_targets_updates_matching_target() {
|
||||||
let mut targets = BucketTargets {
|
let mut targets = BucketTargets {
|
||||||
targets: vec![crate::admin::storage_compat::ecstore::bucket::target::BucketTarget {
|
targets: vec![crate::admin::storage_compat::target::BucketTarget {
|
||||||
arn: "arn:target".to_string(),
|
arn: "arn:target".to_string(),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
}],
|
}],
|
||||||
@@ -2798,10 +2796,10 @@ mod tests {
|
|||||||
let mut status = BucketReplicationResyncStatus::new();
|
let mut status = BucketReplicationResyncStatus::new();
|
||||||
status.targets_map.insert(
|
status.targets_map.insert(
|
||||||
"arn:z".to_string(),
|
"arn:z".to_string(),
|
||||||
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus {
|
crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
|
||||||
resync_id: "rid-z".to_string(),
|
resync_id: "rid-z".to_string(),
|
||||||
last_update: Some(datetime!(2025-01-03 00:00 UTC)),
|
last_update: Some(datetime!(2025-01-03 00:00 UTC)),
|
||||||
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncFailed,
|
resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncFailed,
|
||||||
failed_count: 2,
|
failed_count: 2,
|
||||||
failed_size: 4,
|
failed_size: 4,
|
||||||
bucket: "bucket-z".to_string(),
|
bucket: "bucket-z".to_string(),
|
||||||
@@ -2811,10 +2809,10 @@ mod tests {
|
|||||||
);
|
);
|
||||||
status.targets_map.insert(
|
status.targets_map.insert(
|
||||||
"arn:a".to_string(),
|
"arn:a".to_string(),
|
||||||
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus {
|
crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
|
||||||
resync_id: "rid-a".to_string(),
|
resync_id: "rid-a".to_string(),
|
||||||
last_update: Some(datetime!(2025-01-02 00:00 UTC)),
|
last_update: Some(datetime!(2025-01-02 00:00 UTC)),
|
||||||
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncCompleted,
|
resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncCompleted,
|
||||||
replicated_count: 3,
|
replicated_count: 3,
|
||||||
replicated_size: 9,
|
replicated_size: 9,
|
||||||
bucket: "bucket-a".to_string(),
|
bucket: "bucket-a".to_string(),
|
||||||
@@ -2844,10 +2842,10 @@ mod tests {
|
|||||||
let mut status = BucketReplicationResyncStatus::new();
|
let mut status = BucketReplicationResyncStatus::new();
|
||||||
status.targets_map.insert(
|
status.targets_map.insert(
|
||||||
"arn:z".to_string(),
|
"arn:z".to_string(),
|
||||||
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus {
|
crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
|
||||||
resync_id: "rid-z".to_string(),
|
resync_id: "rid-z".to_string(),
|
||||||
last_update: Some(datetime!(2025-02-03 00:00 UTC)),
|
last_update: Some(datetime!(2025-02-03 00:00 UTC)),
|
||||||
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncFailed,
|
resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncFailed,
|
||||||
failed_count: 2,
|
failed_count: 2,
|
||||||
failed_size: 4,
|
failed_size: 4,
|
||||||
bucket: "bucket-z".to_string(),
|
bucket: "bucket-z".to_string(),
|
||||||
@@ -2857,10 +2855,10 @@ mod tests {
|
|||||||
);
|
);
|
||||||
status.targets_map.insert(
|
status.targets_map.insert(
|
||||||
"arn:a".to_string(),
|
"arn:a".to_string(),
|
||||||
crate::admin::storage_compat::ecstore::bucket::replication::TargetReplicationResyncStatus {
|
crate::admin::storage_compat::replication::TargetReplicationResyncStatus {
|
||||||
resync_id: "rid-a".to_string(),
|
resync_id: "rid-a".to_string(),
|
||||||
last_update: Some(datetime!(2025-02-02 00:00 UTC)),
|
last_update: Some(datetime!(2025-02-02 00:00 UTC)),
|
||||||
resync_status: crate::admin::storage_compat::ecstore::bucket::replication::ResyncStatusType::ResyncCompleted,
|
resync_status: crate::admin::storage_compat::replication::ResyncStatusType::ResyncCompleted,
|
||||||
replicated_count: 3,
|
replicated_count: 3,
|
||||||
replicated_size: 9,
|
replicated_size: 9,
|
||||||
bucket: "bucket-a".to_string(),
|
bucket: "bucket-a".to_string(),
|
||||||
|
|||||||
@@ -12,10 +12,10 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::admin::storage_compat::ecstore::config::com::{STORAGE_CLASS_SUB_SYS, read_config_without_migrate};
|
use crate::admin::storage_compat::com::{STORAGE_CLASS_SUB_SYS, read_config_without_migrate};
|
||||||
use crate::admin::storage_compat::ecstore::config::set_global_storage_class;
|
use crate::admin::storage_compat::get_global_notification_sys;
|
||||||
use crate::admin::storage_compat::ecstore::config::storageclass;
|
use crate::admin::storage_compat::set_global_storage_class;
|
||||||
use crate::admin::storage_compat::ecstore::notification_sys::get_global_notification_sys;
|
use crate::admin::storage_compat::storageclass;
|
||||||
use crate::app::context::resolve_object_store_handle;
|
use crate::app::context::resolve_object_store_handle;
|
||||||
use rustfs_audit::reload_audit_config;
|
use rustfs_audit::reload_audit_config;
|
||||||
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_WEBHOOK_SUB_SYS};
|
use rustfs_config::audit::{AUDIT_MQTT_SUB_SYS, AUDIT_REDIS_DEFAULT_CHANNEL, AUDIT_WEBHOOK_SUB_SYS};
|
||||||
@@ -371,7 +371,7 @@ pub async fn signal_config_snapshot_reload() {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::admin::storage_compat::ecstore::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG};
|
use crate::admin::storage_compat::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_REPLICATION_CONFIG};
|
||||||
use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS;
|
use rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS;
|
||||||
use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES};
|
use rustfs_config::oidc::{OIDC_CLIENT_ID, OIDC_CONFIG_URL, OIDC_SCOPES};
|
||||||
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
|
use rustfs_config::{HEAL_SUB_SYS, SCANNER_SUB_SYS};
|
||||||
@@ -427,7 +427,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn validate_notify_subsystem_config_rejects_invalid_webhook_endpoint() {
|
fn validate_notify_subsystem_config_rejects_invalid_webhook_endpoint() {
|
||||||
crate::admin::storage_compat::ecstore::config::init();
|
crate::admin::storage_compat::init();
|
||||||
let mut config = ServerConfig::new();
|
let mut config = ServerConfig::new();
|
||||||
let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults");
|
let targets = config.0.get_mut(NOTIFY_WEBHOOK_SUB_SYS).expect("notify webhook defaults");
|
||||||
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
|
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
|
||||||
@@ -441,7 +441,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn validate_audit_subsystem_config_rejects_relative_queue_dir() {
|
fn validate_audit_subsystem_config_rejects_relative_queue_dir() {
|
||||||
crate::admin::storage_compat::ecstore::config::init();
|
crate::admin::storage_compat::init();
|
||||||
let mut config = ServerConfig::new();
|
let mut config = ServerConfig::new();
|
||||||
let targets = config.0.get_mut(AUDIT_MQTT_SUB_SYS).expect("audit mqtt defaults");
|
let targets = config.0.get_mut(AUDIT_MQTT_SUB_SYS).expect("audit mqtt defaults");
|
||||||
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
|
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
|
||||||
@@ -456,7 +456,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn validate_identity_openid_config_rejects_missing_openid_scope() {
|
fn validate_identity_openid_config_rejects_missing_openid_scope() {
|
||||||
crate::admin::storage_compat::ecstore::config::init();
|
crate::admin::storage_compat::init();
|
||||||
let mut config = ServerConfig::new();
|
let mut config = ServerConfig::new();
|
||||||
let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults");
|
let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults");
|
||||||
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
|
let kvs = targets.get_mut(DEFAULT_DELIMITER).expect("default target");
|
||||||
@@ -473,7 +473,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn validate_identity_openid_config_rejects_invalid_named_provider_id() {
|
fn validate_identity_openid_config_rejects_invalid_named_provider_id() {
|
||||||
crate::admin::storage_compat::ecstore::config::init();
|
crate::admin::storage_compat::init();
|
||||||
let mut config = ServerConfig::new();
|
let mut config = ServerConfig::new();
|
||||||
let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults");
|
let targets = config.0.get_mut(IDENTITY_OPENID_SUB_SYS).expect("openid defaults");
|
||||||
let default_kvs = targets.get(DEFAULT_DELIMITER).cloned().expect("default target");
|
let default_kvs = targets.get(DEFAULT_DELIMITER).cloned().expect("default target");
|
||||||
|
|||||||
@@ -13,8 +13,8 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::admin::site_replication_identity::{deployment_id_for_endpoint, normalize_peer_map_by_identity_with};
|
use crate::admin::site_replication_identity::{deployment_id_for_endpoint, normalize_peer_map_by_identity_with};
|
||||||
use crate::admin::storage_compat::ecstore::config::com::{read_config, save_config};
|
use crate::admin::storage_compat::Error as StorageError;
|
||||||
use crate::admin::storage_compat::ecstore::error::Error as StorageError;
|
use crate::admin::storage_compat::com::{read_config, save_config};
|
||||||
use crate::app::context::resolve_object_store_handle;
|
use crate::app::context::resolve_object_store_handle;
|
||||||
use rustfs_madmin::PeerInfo;
|
use rustfs_madmin::PeerInfo;
|
||||||
use s3s::{S3Error, S3ErrorCode, S3Result};
|
use s3s::{S3Error, S3ErrorCode, S3Result};
|
||||||
|
|||||||
@@ -12,78 +12,36 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) use rustfs_ecstore::bucket::lifecycle::tier_last_day_stats::DailyAllTierStats;
|
||||||
pub(crate) mod bucket {
|
pub(crate) use rustfs_ecstore::bucket::{
|
||||||
pub(crate) use rustfs_ecstore::bucket::{
|
bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning,
|
||||||
bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning,
|
versioning_sys,
|
||||||
versioning_sys,
|
};
|
||||||
};
|
pub(crate) use rustfs_ecstore::client::admin_handler_utils::AdminError;
|
||||||
}
|
pub(crate) use rustfs_ecstore::config::{com, init, set_global_storage_class, storageclass};
|
||||||
|
pub(crate) use rustfs_ecstore::data_usage::load_data_usage_from_backend;
|
||||||
pub(crate) mod client {
|
pub(crate) use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
|
||||||
pub(crate) use rustfs_ecstore::client::admin_handler_utils;
|
#[cfg(test)]
|
||||||
}
|
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
|
||||||
|
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
|
||||||
pub(crate) mod config {
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_ecstore::config::{com, init, set_global_storage_class, storageclass};
|
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
|
||||||
}
|
pub(crate) use rustfs_ecstore::error::{Error, StorageError};
|
||||||
|
pub(crate) use rustfs_ecstore::global::{
|
||||||
pub(crate) mod data_usage {
|
GLOBAL_BOOT_TIME, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt, get_global_region,
|
||||||
pub(crate) use rustfs_ecstore::data_usage::load_data_usage_from_backend;
|
global_rustfs_port,
|
||||||
}
|
};
|
||||||
|
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
||||||
pub(crate) mod disk {
|
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
|
||||||
pub(crate) use rustfs_ecstore::disk::RUSTFS_META_BUCKET;
|
pub(crate) use rustfs_ecstore::rebalance::{DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_ecstore::disk::endpoint;
|
pub(crate) use rustfs_ecstore::rebalance::{RebalStatus, RebalanceInfo};
|
||||||
}
|
pub(crate) use rustfs_ecstore::rpc::PeerRestClient;
|
||||||
|
pub(crate) use rustfs_ecstore::store::ECStore;
|
||||||
pub(crate) mod endpoints {
|
pub(crate) use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket;
|
||||||
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
|
pub(crate) use rustfs_ecstore::tier::tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_BACKEND_NOT_EMPTY, ERR_TIER_MISSING_CREDENTIALS};
|
||||||
#[cfg(test)]
|
pub(crate) use rustfs_ecstore::tier::tier_admin::TierCreds;
|
||||||
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
|
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierType};
|
||||||
}
|
pub(crate) use rustfs_ecstore::tier::tier_handlers::{
|
||||||
|
ERR_TIER_ALREADY_EXISTS, ERR_TIER_CONNECT_ERR, ERR_TIER_INVALID_CREDENTIALS, ERR_TIER_NAME_NOT_UPPERCASE, ERR_TIER_NOT_FOUND,
|
||||||
pub(crate) mod error {
|
};
|
||||||
pub(crate) use rustfs_ecstore::error::{Error, StorageError};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod global {
|
|
||||||
pub(crate) use rustfs_ecstore::global::{
|
|
||||||
GLOBAL_BOOT_TIME, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints_opt, get_global_region,
|
|
||||||
global_rustfs_port,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod metrics_realtime {
|
|
||||||
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod notification_sys {
|
|
||||||
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod rebalance {
|
|
||||||
pub(crate) use rustfs_ecstore::rebalance::{
|
|
||||||
DiskStat, RebalSaveOpt, RebalanceCleanupWarnings, RebalanceMeta, RebalanceStats,
|
|
||||||
};
|
|
||||||
#[cfg(test)]
|
|
||||||
pub(crate) use rustfs_ecstore::rebalance::{RebalStatus, RebalanceInfo};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod rpc {
|
|
||||||
pub(crate) use rustfs_ecstore::rpc::PeerRestClient;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod store {
|
|
||||||
pub(crate) use rustfs_ecstore::store::ECStore;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod store_utils {
|
|
||||||
pub(crate) use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod tier {
|
|
||||||
pub(crate) use rustfs_ecstore::tier::{tier, tier_admin, tier_config, tier_handlers};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -15,13 +15,11 @@
|
|||||||
//! Admin application use-case contracts.
|
//! Admin application use-case contracts.
|
||||||
|
|
||||||
use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
|
use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
|
||||||
use crate::app::storage_compat::ecstore::admin_server_info::get_server_info;
|
use crate::app::storage_compat::ECStore;
|
||||||
use crate::app::storage_compat::ecstore::data_usage::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend};
|
use crate::app::storage_compat::EndpointServerPools;
|
||||||
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools;
|
use crate::app::storage_compat::get_server_info;
|
||||||
use crate::app::storage_compat::ecstore::pools::{
|
use crate::app::storage_compat::{PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free};
|
||||||
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
use crate::app::storage_compat::{apply_bucket_usage_memory_overlay, load_data_usage_from_backend};
|
||||||
};
|
|
||||||
use crate::app::storage_compat::ecstore::store::ECStore;
|
|
||||||
use crate::capacity::resolve_admin_used_capacity;
|
use crate::capacity::resolve_admin_used_capacity;
|
||||||
use crate::error::ApiError;
|
use crate::error::ApiError;
|
||||||
use crate::server::{DependencyReadiness, collect_dependency_readiness as collect_runtime_dependency_readiness};
|
use crate::server::{DependencyReadiness, collect_dependency_readiness as collect_runtime_dependency_readiness};
|
||||||
@@ -318,7 +316,7 @@ impl DefaultAdminUsecase {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::app::storage_compat::ecstore::pools::{PoolDecommissionInfo, PoolStatus};
|
use crate::app::storage_compat::{PoolDecommissionInfo, PoolStatus};
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -20,7 +20,11 @@ use crate::admin::handlers::site_replication::{
|
|||||||
use crate::app::context::{
|
use crate::app::context::{
|
||||||
AppContext, default_notify_interface, get_global_app_context, resolve_object_store_handle_for_context,
|
AppContext, default_notify_interface, get_global_app_context, resolve_object_store_handle_for_context,
|
||||||
};
|
};
|
||||||
use crate::app::storage_compat::ecstore::bucket::{
|
use crate::app::storage_compat::ECStore;
|
||||||
|
use crate::app::storage_compat::StorageError;
|
||||||
|
use crate::app::storage_compat::get_global_notification_sys;
|
||||||
|
use crate::app::storage_compat::object_api_utils::to_s3s_etag;
|
||||||
|
use crate::app::storage_compat::{
|
||||||
bucket_target_sys::BucketTargetSys,
|
bucket_target_sys::BucketTargetSys,
|
||||||
lifecycle::bucket_lifecycle_ops::{
|
lifecycle::bucket_lifecycle_ops::{
|
||||||
enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, validate_transition_tier,
|
enqueue_expiry_for_existing_objects, enqueue_transition_for_existing_objects, validate_transition_tier,
|
||||||
@@ -38,10 +42,6 @@ use crate::app::storage_compat::ecstore::bucket::{
|
|||||||
versioning::VersioningApi,
|
versioning::VersioningApi,
|
||||||
versioning_sys::BucketVersioningSys,
|
versioning_sys::BucketVersioningSys,
|
||||||
};
|
};
|
||||||
use crate::app::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
|
|
||||||
use crate::app::storage_compat::ecstore::error::StorageError;
|
|
||||||
use crate::app::storage_compat::ecstore::notification_sys::get_global_notification_sys;
|
|
||||||
use crate::app::storage_compat::ecstore::store::ECStore;
|
|
||||||
use crate::auth::get_condition_values_with_client_info;
|
use crate::auth::get_condition_values_with_client_info;
|
||||||
use crate::error::ApiError;
|
use crate::error::ApiError;
|
||||||
use crate::server::RemoteAddr;
|
use crate::server::RemoteAddr;
|
||||||
@@ -1628,9 +1628,7 @@ impl DefaultBucketUsecase {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Err(err) =
|
if let Err(err) = crate::app::storage_compat::lifecycle::lifecycle::Lifecycle::validate(&input_cfg, &rcfg).await {
|
||||||
crate::app::storage_compat::ecstore::bucket::lifecycle::lifecycle::Lifecycle::validate(&input_cfg, &rcfg).await
|
|
||||||
{
|
|
||||||
return Err(s3_error!(InvalidArgument, "{err}"));
|
return Err(s3_error!(InvalidArgument, "{err}"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2280,7 +2278,7 @@ mod tests {
|
|||||||
BucketTargets {
|
BucketTargets {
|
||||||
targets: arns
|
targets: arns
|
||||||
.iter()
|
.iter()
|
||||||
.map(|arn| crate::app::storage_compat::ecstore::bucket::target::BucketTarget {
|
.map(|arn| crate::app::storage_compat::target::BucketTarget {
|
||||||
arn: (*arn).to_string(),
|
arn: (*arn).to_string(),
|
||||||
target_type: BucketTargetType::ReplicationService,
|
target_type: BucketTargetType::ReplicationService,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
|
|||||||
@@ -12,12 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::app::storage_compat::ecstore::{
|
use crate::app::storage_compat::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, metadata_sys};
|
||||||
bucket::metadata_sys,
|
|
||||||
disk::endpoint::Endpoint,
|
|
||||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
|
||||||
store::ECStore,
|
|
||||||
};
|
|
||||||
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
|
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
|
||||||
use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager};
|
use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager};
|
||||||
use rustfs_storage_api::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _};
|
use rustfs_storage_api::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _};
|
||||||
@@ -82,7 +77,7 @@ async fn setup_capacity_dirty_scope_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
||||||
crate::app::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
|
crate::app::storage_compat::init_local_disks(endpoint_pools.clone())
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
|
|||||||
@@ -17,11 +17,11 @@ use super::handles::{
|
|||||||
default_bucket_metadata_interface, default_endpoints_interface, default_kms_runtime_interface,
|
default_bucket_metadata_interface, default_endpoints_interface, default_kms_runtime_interface,
|
||||||
default_server_config_interface, default_tier_config_interface,
|
default_server_config_interface, default_tier_config_interface,
|
||||||
};
|
};
|
||||||
use crate::app::storage_compat::ecstore::bucket::metadata_sys::BucketMetadataSys;
|
use crate::app::storage_compat::ECStore;
|
||||||
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools;
|
use crate::app::storage_compat::EndpointServerPools;
|
||||||
use crate::app::storage_compat::ecstore::new_object_layer_fn;
|
use crate::app::storage_compat::TierConfigMgr;
|
||||||
use crate::app::storage_compat::ecstore::store::ECStore;
|
use crate::app::storage_compat::metadata_sys::BucketMetadataSys;
|
||||||
use crate::app::storage_compat::ecstore::tier::tier::TierConfigMgr;
|
use crate::app::storage_compat::new_object_layer_fn;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use crate::config::RustFSBufferConfig;
|
use crate::config::RustFSBufferConfig;
|
||||||
use rustfs_config::server_config::Config;
|
use rustfs_config::server_config::Config;
|
||||||
@@ -126,10 +126,10 @@ mod tests {
|
|||||||
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
|
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
|
||||||
ServerConfigInterface, TierConfigInterface,
|
ServerConfigInterface, TierConfigInterface,
|
||||||
};
|
};
|
||||||
use crate::app::storage_compat::ecstore::disk::endpoint::Endpoint;
|
use crate::app::storage_compat::Endpoint;
|
||||||
use crate::app::storage_compat::ecstore::endpoints::{Endpoints, PoolEndpoints};
|
use crate::app::storage_compat::init_local_disks;
|
||||||
use crate::app::storage_compat::ecstore::new_object_layer_fn;
|
use crate::app::storage_compat::new_object_layer_fn;
|
||||||
use crate::app::storage_compat::ecstore::store::init_local_disks;
|
use crate::app::storage_compat::{Endpoints, PoolEndpoints};
|
||||||
use crate::config::{RustFSBufferConfig, WorkloadProfile};
|
use crate::config::{RustFSBufferConfig, WorkloadProfile};
|
||||||
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
|
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ use super::interfaces::{
|
|||||||
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
|
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
|
||||||
NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface,
|
NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface,
|
||||||
};
|
};
|
||||||
use crate::app::storage_compat::ecstore::{set_object_store_resolver, store::ECStore};
|
use crate::app::storage_compat::{ECStore, set_object_store_resolver};
|
||||||
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
|
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
|
||||||
use rustfs_kms::KmsServiceManager;
|
use rustfs_kms::KmsServiceManager;
|
||||||
use std::sync::{Arc, OnceLock};
|
use std::sync::{Arc, OnceLock};
|
||||||
|
|||||||
@@ -16,10 +16,10 @@ use super::interfaces::{
|
|||||||
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
|
BucketMetadataInterface, BufferConfigInterface, EndpointsInterface, IamInterface, KmsInterface, KmsRuntimeInterface,
|
||||||
NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface,
|
NotifyInterface, RegionInterface, ServerConfigInterface, TierConfigInterface,
|
||||||
};
|
};
|
||||||
use crate::app::storage_compat::ecstore::bucket::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys};
|
use crate::app::storage_compat::EndpointServerPools;
|
||||||
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools;
|
use crate::app::storage_compat::TierConfigMgr;
|
||||||
use crate::app::storage_compat::ecstore::global::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr};
|
use crate::app::storage_compat::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys};
|
||||||
use crate::app::storage_compat::ecstore::tier::tier::TierConfigMgr;
|
use crate::app::storage_compat::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr};
|
||||||
use crate::config::{RustFSBufferConfig, get_global_buffer_config};
|
use crate::config::{RustFSBufferConfig, get_global_buffer_config};
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use rustfs_config::server_config::Config;
|
use rustfs_config::server_config::Config;
|
||||||
|
|||||||
@@ -12,9 +12,9 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::app::storage_compat::ecstore::bucket::metadata_sys::BucketMetadataSys;
|
use crate::app::storage_compat::EndpointServerPools;
|
||||||
use crate::app::storage_compat::ecstore::endpoints::EndpointServerPools;
|
use crate::app::storage_compat::TierConfigMgr;
|
||||||
use crate::app::storage_compat::ecstore::tier::tier::TierConfigMgr;
|
use crate::app::storage_compat::metadata_sys::BucketMetadataSys;
|
||||||
use crate::config::RustFSBufferConfig;
|
use crate::config::RustFSBufferConfig;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use rustfs_config::server_config::Config;
|
use rustfs_config::server_config::Config;
|
||||||
|
|||||||
@@ -14,19 +14,13 @@
|
|||||||
|
|
||||||
use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase};
|
use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase};
|
||||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||||
use crate::app::storage_compat::ecstore::{
|
use crate::app::storage_compat::{
|
||||||
bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG},
|
ECStore, Endpoint, EndpointServerPools, Endpoints, GLOBAL_TierConfigMgr, PoolEndpoints, TierConfig, TierType, WarmBackend,
|
||||||
bucket::metadata_sys,
|
WarmBackendGetOpts,
|
||||||
client::object_api_utils::to_s3s_etag,
|
metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG},
|
||||||
client::transition_api::{ReadCloser, ReaderImpl},
|
metadata_sys,
|
||||||
disk::endpoint::Endpoint,
|
object_api_utils::to_s3s_etag,
|
||||||
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
|
transition_api::{ReadCloser, ReaderImpl},
|
||||||
global::GLOBAL_TierConfigMgr,
|
|
||||||
store::ECStore,
|
|
||||||
tier::{
|
|
||||||
tier_config::{TierConfig, TierType},
|
|
||||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
use crate::storage::ecfs::FS;
|
use crate::storage::ecfs::FS;
|
||||||
use crate::storage::{
|
use crate::storage::{
|
||||||
@@ -113,7 +107,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
|||||||
|
|
||||||
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
let endpoint_pools = EndpointServerPools(vec![pool_endpoints]);
|
||||||
|
|
||||||
crate::app::storage_compat::ecstore::store::init_local_disks(endpoint_pools.clone())
|
crate::app::storage_compat::init_local_disks(endpoint_pools.clone())
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
@@ -132,7 +126,7 @@ async fn setup_test_env() -> (Vec<PathBuf>, Arc<ECStore>) {
|
|||||||
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
let buckets = buckets_list.into_iter().map(|v| v.name).collect();
|
||||||
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
metadata_sys::init_bucket_metadata_sys(ecstore.clone(), buckets).await;
|
||||||
|
|
||||||
crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
|
crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::init_background_expiry(ecstore.clone()).await;
|
||||||
|
|
||||||
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
|
let _ = GLOBAL_ENV.set((disk_paths.clone(), ecstore.clone()));
|
||||||
|
|
||||||
@@ -932,7 +926,7 @@ async fn delete_transitioned_object_removes_remote_tier_copy_via_usecase() {
|
|||||||
.expect("Failed to set lifecycle configuration");
|
.expect("Failed to set lifecycle configuration");
|
||||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||||
|
|
||||||
crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||||
ecstore.clone(),
|
ecstore.clone(),
|
||||||
bucket.as_str(),
|
bucket.as_str(),
|
||||||
)
|
)
|
||||||
@@ -992,7 +986,7 @@ async fn lifecycle_transition_marks_dirty_disks_for_capacity_manager() {
|
|||||||
.expect("Failed to set lifecycle configuration");
|
.expect("Failed to set lifecycle configuration");
|
||||||
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
let _ = upload_test_object(&ecstore, bucket.as_str(), object, payload).await;
|
||||||
|
|
||||||
crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects(
|
||||||
ecstore.clone(),
|
ecstore.clone(),
|
||||||
bucket.as_str(),
|
bucket.as_str(),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -16,22 +16,22 @@
|
|||||||
|
|
||||||
use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
|
use crate::app::context::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
|
||||||
use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write};
|
use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write};
|
||||||
use crate::app::storage_compat::ecstore::bucket::quota::checker::QuotaChecker;
|
use crate::app::storage_compat::ECStore;
|
||||||
use crate::app::storage_compat::ecstore::bucket::{
|
use crate::app::storage_compat::is_disk_compressible;
|
||||||
|
use crate::app::storage_compat::is_valid_storage_class;
|
||||||
|
use crate::app::storage_compat::object_api_utils::to_s3s_etag;
|
||||||
|
use crate::app::storage_compat::quota::checker::QuotaChecker;
|
||||||
|
#[cfg(test)]
|
||||||
|
use crate::app::storage_compat::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
|
||||||
|
use crate::app::storage_compat::{HashReader, WritePlan};
|
||||||
|
use crate::app::storage_compat::{StorageError, is_err_object_not_found, is_err_version_not_found};
|
||||||
|
use crate::app::storage_compat::{
|
||||||
lifecycle::{bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::enqueue_transition_immediate},
|
lifecycle::{bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::enqueue_transition_immediate},
|
||||||
metadata_sys,
|
metadata_sys,
|
||||||
quota::QuotaOperation,
|
quota::QuotaOperation,
|
||||||
replication::{get_must_replicate_options, must_replicate, schedule_replication},
|
replication::{get_must_replicate_options, must_replicate, schedule_replication},
|
||||||
versioning_sys::BucketVersioningSys,
|
versioning_sys::BucketVersioningSys,
|
||||||
};
|
};
|
||||||
use crate::app::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
|
|
||||||
use crate::app::storage_compat::ecstore::compress::is_disk_compressible;
|
|
||||||
use crate::app::storage_compat::ecstore::error::{StorageError, is_err_object_not_found, is_err_version_not_found};
|
|
||||||
#[cfg(test)]
|
|
||||||
use crate::app::storage_compat::ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
|
|
||||||
use crate::app::storage_compat::ecstore::rio::{HashReader, WritePlan};
|
|
||||||
use crate::app::storage_compat::ecstore::set_disk::is_valid_storage_class;
|
|
||||||
use crate::app::storage_compat::ecstore::store::ECStore;
|
|
||||||
use crate::capacity::record_capacity_write;
|
use crate::capacity::record_capacity_write;
|
||||||
use crate::error::ApiError;
|
use crate::error::ApiError;
|
||||||
use crate::storage::access::has_bypass_governance_header;
|
use crate::storage::access::has_bypass_governance_header;
|
||||||
@@ -479,7 +479,7 @@ impl DefaultMultipartUsecase {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
// Update quota tracking after successful multipart upload
|
// Update quota tracking after successful multipart upload
|
||||||
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_write_memory(
|
crate::app::storage_compat::record_bucket_object_write_memory(
|
||||||
&bucket,
|
&bucket,
|
||||||
previous_current_size,
|
previous_current_size,
|
||||||
obj_info.size.max(0) as u64,
|
obj_info.size.max(0) as u64,
|
||||||
@@ -672,7 +672,7 @@ impl DefaultMultipartUsecase {
|
|||||||
rustfs_utils::http::insert_str(
|
rustfs_utils::http::insert_str(
|
||||||
&mut metadata,
|
&mut metadata,
|
||||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||||
crate::app::storage_compat::ecstore::rio::compression_metadata_value(CompressionAlgorithm::default()),
|
crate::app::storage_compat::compression_metadata_value(CompressionAlgorithm::default()),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -891,18 +891,13 @@ impl DefaultMultipartUsecase {
|
|||||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
|
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
|
||||||
let ssec_write = match ssec_material.key_kind {
|
let ssec_write = match ssec_material.key_kind {
|
||||||
crate::storage::sse::EncryptionKeyKind::Object => {
|
crate::storage::sse::EncryptionKeyKind::Object => {
|
||||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
|
crate::app::storage_compat::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
|
||||||
ssec_material.key_bytes,
|
|
||||||
part_id as u32,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
crate::storage::sse::EncryptionKeyKind::Direct => {
|
|
||||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
|
|
||||||
ssec_material.key_bytes,
|
|
||||||
ssec_material.base_nonce,
|
|
||||||
part_id,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
|
||||||
|
ssec_material.key_bytes,
|
||||||
|
ssec_material.base_nonce,
|
||||||
|
part_id,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
write_plan = write_plan.with_encryption(ssec_write);
|
write_plan = write_plan.with_encryption(ssec_write);
|
||||||
(Some(ssec_material.server_side_encryption), ssec_material.kms_key_id)
|
(Some(ssec_material.server_side_encryption), ssec_material.kms_key_id)
|
||||||
@@ -918,18 +913,13 @@ impl DefaultMultipartUsecase {
|
|||||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
|
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
|
||||||
let managed_write = match managed_material.key_kind {
|
let managed_write = match managed_material.key_kind {
|
||||||
crate::storage::sse::EncryptionKeyKind::Object => {
|
crate::storage::sse::EncryptionKeyKind::Object => {
|
||||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
|
crate::app::storage_compat::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
|
||||||
managed_material.key_bytes,
|
|
||||||
part_id as u32,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
crate::storage::sse::EncryptionKeyKind::Direct => {
|
|
||||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
|
|
||||||
managed_material.key_bytes,
|
|
||||||
managed_material.base_nonce,
|
|
||||||
part_id,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
|
||||||
|
managed_material.key_bytes,
|
||||||
|
managed_material.base_nonce,
|
||||||
|
part_id,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
write_plan = write_plan.with_encryption(managed_write);
|
write_plan = write_plan.with_encryption(managed_write);
|
||||||
(Some(server_side_encryption), ssekms_key_id)
|
(Some(server_side_encryption), ssekms_key_id)
|
||||||
@@ -1244,18 +1234,13 @@ impl DefaultMultipartUsecase {
|
|||||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
|
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
|
||||||
let ssec_write = match ssec_material.key_kind {
|
let ssec_write = match ssec_material.key_kind {
|
||||||
crate::storage::sse::EncryptionKeyKind::Object => {
|
crate::storage::sse::EncryptionKeyKind::Object => {
|
||||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
|
crate::app::storage_compat::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
|
||||||
ssec_material.key_bytes,
|
|
||||||
part_id as u32,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
crate::storage::sse::EncryptionKeyKind::Direct => {
|
|
||||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
|
|
||||||
ssec_material.key_bytes,
|
|
||||||
ssec_material.base_nonce,
|
|
||||||
part_id,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
|
||||||
|
ssec_material.key_bytes,
|
||||||
|
ssec_material.base_nonce,
|
||||||
|
part_id,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
write_plan = write_plan.with_encryption(ssec_write);
|
write_plan = write_plan.with_encryption(ssec_write);
|
||||||
(
|
(
|
||||||
@@ -1275,18 +1260,13 @@ impl DefaultMultipartUsecase {
|
|||||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
|
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
|
||||||
let managed_write = match managed_material.key_kind {
|
let managed_write = match managed_material.key_kind {
|
||||||
crate::storage::sse::EncryptionKeyKind::Object => {
|
crate::storage::sse::EncryptionKeyKind::Object => {
|
||||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
|
crate::app::storage_compat::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
|
||||||
managed_material.key_bytes,
|
|
||||||
part_id as u32,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
crate::storage::sse::EncryptionKeyKind::Direct => {
|
|
||||||
crate::app::storage_compat::ecstore::rio::WriteEncryption::multipart(
|
|
||||||
managed_material.key_bytes,
|
|
||||||
managed_material.base_nonce,
|
|
||||||
part_id,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
crate::storage::sse::EncryptionKeyKind::Direct => crate::app::storage_compat::WriteEncryption::multipart(
|
||||||
|
managed_material.key_bytes,
|
||||||
|
managed_material.base_nonce,
|
||||||
|
part_id,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
write_plan = write_plan.with_encryption(managed_write);
|
write_plan = write_plan.with_encryption(managed_write);
|
||||||
(Some(server_side_encryption), ssekms_key_id, mp_info.user_defined.clone())
|
(Some(server_side_encryption), ssekms_key_id, mp_info.user_defined.clone())
|
||||||
|
|||||||
@@ -48,8 +48,18 @@ use metrics::{counter, histogram};
|
|||||||
use pin_project_lite::pin_project;
|
use pin_project_lite::pin_project;
|
||||||
use rustfs_object_capacity::capacity_manager::get_capacity_manager;
|
use rustfs_object_capacity::capacity_manager::get_capacity_manager;
|
||||||
// Performance metrics recording (with zero-copy-metrics integration)
|
// Performance metrics recording (with zero-copy-metrics integration)
|
||||||
use crate::app::storage_compat::ecstore::bucket::quota::checker::QuotaChecker;
|
use crate::app::storage_compat::ECStore;
|
||||||
use crate::app::storage_compat::ecstore::bucket::{
|
use crate::app::storage_compat::object_api_utils::to_s3s_etag;
|
||||||
|
use crate::app::storage_compat::quota::checker::QuotaChecker;
|
||||||
|
use crate::app::storage_compat::storageclass;
|
||||||
|
use crate::app::storage_compat::{DiskError, is_all_buckets_not_found};
|
||||||
|
use crate::app::storage_compat::{DynReader, HashReader, WritePlan, wrap_reader};
|
||||||
|
use crate::app::storage_compat::{
|
||||||
|
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||||
|
};
|
||||||
|
use crate::app::storage_compat::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||||
|
use crate::app::storage_compat::{get_lock_acquire_timeout, is_valid_storage_class};
|
||||||
|
use crate::app::storage_compat::{
|
||||||
lifecycle::{
|
lifecycle::{
|
||||||
bucket_lifecycle_audit::LcEventSrc,
|
bucket_lifecycle_audit::LcEventSrc,
|
||||||
bucket_lifecycle_ops::{RestoreRequestOps, enqueue_transition_immediate, post_restore_opts},
|
bucket_lifecycle_ops::{RestoreRequestOps, enqueue_transition_immediate, post_restore_opts},
|
||||||
@@ -69,16 +79,6 @@ use crate::app::storage_compat::ecstore::bucket::{
|
|||||||
versioning::VersioningApi,
|
versioning::VersioningApi,
|
||||||
versioning_sys::BucketVersioningSys,
|
versioning_sys::BucketVersioningSys,
|
||||||
};
|
};
|
||||||
use crate::app::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
|
|
||||||
use crate::app::storage_compat::ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
|
||||||
use crate::app::storage_compat::ecstore::config::storageclass;
|
|
||||||
use crate::app::storage_compat::ecstore::disk::{error::DiskError, error_reduce::is_all_buckets_not_found};
|
|
||||||
use crate::app::storage_compat::ecstore::error::{
|
|
||||||
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
|
||||||
};
|
|
||||||
use crate::app::storage_compat::ecstore::rio::{DynReader, HashReader, WritePlan, wrap_reader};
|
|
||||||
use crate::app::storage_compat::ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
|
|
||||||
use crate::app::storage_compat::ecstore::store::ECStore;
|
|
||||||
use rustfs_concurrency::GetObjectQueueSnapshot;
|
use rustfs_concurrency::GetObjectQueueSnapshot;
|
||||||
use rustfs_filemeta::{
|
use rustfs_filemeta::{
|
||||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType,
|
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType,
|
||||||
@@ -276,12 +276,12 @@ async fn enqueue_transitioned_delete_cleanup(
|
|||||||
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup);
|
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup);
|
||||||
|
|
||||||
let je = if opts.delete_prefix {
|
let je = if opts.delete_prefix {
|
||||||
crate::app::storage_compat::ecstore::bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(
|
crate::app::storage_compat::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(
|
||||||
&existing.transitioned_object,
|
&existing.transitioned_object,
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok());
|
let version_id = opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok());
|
||||||
crate::app::storage_compat::ecstore::bucket::lifecycle::tier_sweeper::transitioned_delete_journal_entry(
|
crate::app::storage_compat::lifecycle::tier_sweeper::transitioned_delete_journal_entry(
|
||||||
version_id,
|
version_id,
|
||||||
opts.versioned,
|
opts.versioned,
|
||||||
opts.version_suspended,
|
opts.version_suspended,
|
||||||
@@ -292,10 +292,9 @@ async fn enqueue_transitioned_delete_cleanup(
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
|
|
||||||
crate::app::storage_compat::ecstore::bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(store, &je)
|
crate::app::storage_compat::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(store, &je).await?;
|
||||||
.await?;
|
|
||||||
|
|
||||||
let mut expiry_state = crate::app::storage_compat::ecstore::bucket::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
|
let mut expiry_state = crate::app::storage_compat::lifecycle::bucket_lifecycle_ops::GLOBAL_ExpiryState
|
||||||
.write()
|
.write()
|
||||||
.await;
|
.await;
|
||||||
if let Err(err) = expiry_state.enqueue_tier_journal_entry(&je).await {
|
if let Err(err) = expiry_state.enqueue_tier_journal_entry(&je).await {
|
||||||
@@ -1540,7 +1539,7 @@ impl DefaultObjectUsecase {
|
|||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
async fn prepare_get_object_read(
|
async fn prepare_get_object_read(
|
||||||
req: &S3Request<GetObjectInput>,
|
req: &S3Request<GetObjectInput>,
|
||||||
store: &crate::app::storage_compat::ecstore::store::ECStore,
|
store: &crate::app::storage_compat::ECStore,
|
||||||
manager: &ConcurrencyManager,
|
manager: &ConcurrencyManager,
|
||||||
bucket: &str,
|
bucket: &str,
|
||||||
key: &str,
|
key: &str,
|
||||||
@@ -2134,7 +2133,7 @@ impl DefaultObjectUsecase {
|
|||||||
insert_str(
|
insert_str(
|
||||||
&mut metadata,
|
&mut metadata,
|
||||||
SUFFIX_COMPRESSION,
|
SUFFIX_COMPRESSION,
|
||||||
crate::app::storage_compat::ecstore::rio::compression_metadata_value(algorithm),
|
crate::app::storage_compat::compression_metadata_value(algorithm),
|
||||||
);
|
);
|
||||||
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
|
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
|
||||||
|
|
||||||
@@ -2149,7 +2148,7 @@ impl DefaultObjectUsecase {
|
|||||||
insert_str(
|
insert_str(
|
||||||
&mut opts.user_defined,
|
&mut opts.user_defined,
|
||||||
SUFFIX_COMPRESSION,
|
SUFFIX_COMPRESSION,
|
||||||
crate::app::storage_compat::ecstore::rio::compression_metadata_value(algorithm),
|
crate::app::storage_compat::compression_metadata_value(algorithm),
|
||||||
);
|
);
|
||||||
insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string());
|
insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string());
|
||||||
|
|
||||||
@@ -2341,7 +2340,7 @@ impl DefaultObjectUsecase {
|
|||||||
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
|
maybe_enqueue_transition_immediate(&obj_info, LcEventSrc::S3PutObject).await;
|
||||||
|
|
||||||
// Fast in-memory update for immediate quota and admin usage consistency
|
// Fast in-memory update for immediate quota and admin usage consistency
|
||||||
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_write_memory(
|
crate::app::storage_compat::record_bucket_object_write_memory(
|
||||||
&bucket,
|
&bucket,
|
||||||
previous_current_size,
|
previous_current_size,
|
||||||
obj_info.size.max(0) as u64,
|
obj_info.size.max(0) as u64,
|
||||||
@@ -3211,7 +3210,7 @@ impl DefaultObjectUsecase {
|
|||||||
insert_str(
|
insert_str(
|
||||||
&mut compress_metadata,
|
&mut compress_metadata,
|
||||||
SUFFIX_COMPRESSION,
|
SUFFIX_COMPRESSION,
|
||||||
crate::app::storage_compat::ecstore::rio::compression_metadata_value(CompressionAlgorithm::default()),
|
crate::app::storage_compat::compression_metadata_value(CompressionAlgorithm::default()),
|
||||||
);
|
);
|
||||||
insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
|
insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
|
||||||
} else {
|
} else {
|
||||||
@@ -3304,12 +3303,8 @@ impl DefaultObjectUsecase {
|
|||||||
|
|
||||||
// Update quota tracking after successful copy
|
// Update quota tracking after successful copy
|
||||||
if has_bucket_metadata {
|
if has_bucket_metadata {
|
||||||
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_write_memory(
|
crate::app::storage_compat::record_bucket_object_write_memory(&bucket, previous_current_size, oi.size.max(0) as u64)
|
||||||
&bucket,
|
.await;
|
||||||
previous_current_size,
|
|
||||||
oi.size.max(0) as u64,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let raw_dest_version = oi.version_id.map(|v| v.to_string());
|
let raw_dest_version = oi.version_id.map(|v| v.to_string());
|
||||||
@@ -3586,7 +3581,7 @@ impl DefaultObjectUsecase {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
let size = object_sizes[i].max(0) as u64;
|
let size = object_sizes[i].max(0) as u64;
|
||||||
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_delete_memory(
|
crate::app::storage_compat::record_bucket_object_delete_memory(
|
||||||
&bucket,
|
&bucket,
|
||||||
size,
|
size,
|
||||||
existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
|
existing_object_infos[i].is_some() && object_to_delete[i].version_id.is_none(),
|
||||||
@@ -3824,7 +3819,7 @@ impl DefaultObjectUsecase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Fast in-memory update for immediate quota and admin usage consistency
|
// Fast in-memory update for immediate quota and admin usage consistency
|
||||||
crate::app::storage_compat::ecstore::data_usage::record_bucket_object_delete_memory(
|
crate::app::storage_compat::record_bucket_object_delete_memory(
|
||||||
&bucket,
|
&bucket,
|
||||||
obj_info.size.max(0) as u64,
|
obj_info.size.max(0) as u64,
|
||||||
opts.version_id.is_none(),
|
opts.version_id.is_none(),
|
||||||
@@ -4757,7 +4752,7 @@ impl DefaultObjectUsecase {
|
|||||||
insert_str(
|
insert_str(
|
||||||
&mut metadata,
|
&mut metadata,
|
||||||
SUFFIX_COMPRESSION,
|
SUFFIX_COMPRESSION,
|
||||||
crate::app::storage_compat::ecstore::rio::compression_metadata_value(algorithm),
|
crate::app::storage_compat::compression_metadata_value(algorithm),
|
||||||
);
|
);
|
||||||
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
|
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
|
||||||
|
|
||||||
|
|||||||
@@ -12,96 +12,50 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) use rustfs_ecstore::admin_server_info::get_server_info;
|
||||||
pub(crate) use rustfs_ecstore::global::{new_object_layer_fn, set_object_store_resolver};
|
pub(crate) use rustfs_ecstore::bucket::{
|
||||||
|
bucket_target_sys, lifecycle, metadata, metadata_sys, object_lock, policy_sys, quota, replication, tagging, target, utils,
|
||||||
pub(crate) mod admin_server_info {
|
versioning, versioning_sys,
|
||||||
pub(crate) use rustfs_ecstore::admin_server_info::get_server_info;
|
};
|
||||||
}
|
pub(crate) use rustfs_ecstore::client::object_api_utils;
|
||||||
|
#[cfg(test)]
|
||||||
pub(crate) mod bucket {
|
pub(crate) use rustfs_ecstore::client::transition_api;
|
||||||
pub(crate) use rustfs_ecstore::bucket::{
|
pub(crate) use rustfs_ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||||
bucket_target_sys, lifecycle, metadata, metadata_sys, object_lock, policy_sys, quota, replication, tagging, target,
|
pub(crate) use rustfs_ecstore::config::storageclass;
|
||||||
utils, versioning, versioning_sys,
|
pub(crate) use rustfs_ecstore::data_usage::{
|
||||||
};
|
apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory,
|
||||||
}
|
record_bucket_object_write_memory,
|
||||||
|
};
|
||||||
pub(crate) mod client {
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_ecstore::client::object_api_utils;
|
pub(crate) use rustfs_ecstore::disk::endpoint::Endpoint;
|
||||||
#[cfg(test)]
|
pub(crate) use rustfs_ecstore::disk::error::DiskError;
|
||||||
pub(crate) use rustfs_ecstore::client::transition_api;
|
pub(crate) use rustfs_ecstore::disk::error_reduce::is_all_buckets_not_found;
|
||||||
}
|
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
|
||||||
|
#[cfg(test)]
|
||||||
pub(crate) mod compress {
|
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
|
||||||
pub(crate) use rustfs_ecstore::compress::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
pub(crate) use rustfs_ecstore::error::{
|
||||||
}
|
Error, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||||
|
};
|
||||||
pub(crate) mod config {
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_ecstore::config::storageclass;
|
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
|
||||||
}
|
pub(crate) use rustfs_ecstore::global::{
|
||||||
|
get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr, new_object_layer_fn, set_object_store_resolver,
|
||||||
pub(crate) mod data_usage {
|
};
|
||||||
pub(crate) use rustfs_ecstore::data_usage::{
|
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
|
||||||
apply_bucket_usage_memory_overlay, load_data_usage_from_backend, record_bucket_object_delete_memory,
|
pub(crate) use rustfs_ecstore::pools::{
|
||||||
record_bucket_object_write_memory,
|
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
||||||
};
|
};
|
||||||
}
|
#[cfg(test)]
|
||||||
|
pub(crate) use rustfs_ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader};
|
||||||
pub(crate) mod disk {
|
pub(crate) use rustfs_ecstore::rio::{
|
||||||
#[cfg(test)]
|
DynReader, HashReader, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader,
|
||||||
pub(crate) use rustfs_ecstore::disk::endpoint;
|
};
|
||||||
pub(crate) use rustfs_ecstore::disk::{error, error_reduce};
|
pub(crate) use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
|
||||||
}
|
pub(crate) use rustfs_ecstore::store::ECStore;
|
||||||
|
#[cfg(test)]
|
||||||
pub(crate) mod endpoints {
|
pub(crate) use rustfs_ecstore::store::init_local_disks;
|
||||||
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
|
pub(crate) use rustfs_ecstore::tier::tier::TierConfigMgr;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
|
pub(crate) use rustfs_ecstore::tier::tier_config::{TierConfig, TierType};
|
||||||
}
|
#[cfg(test)]
|
||||||
|
pub(crate) use rustfs_ecstore::tier::warm_backend::{WarmBackend, WarmBackendGetOpts};
|
||||||
pub(crate) mod error {
|
|
||||||
pub(crate) use rustfs_ecstore::error::{
|
|
||||||
Error, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod global {
|
|
||||||
#[cfg(test)]
|
|
||||||
pub(crate) use rustfs_ecstore::global::GLOBAL_TierConfigMgr;
|
|
||||||
pub(crate) use rustfs_ecstore::global::{get_global_endpoints_opt, get_global_region, get_global_tier_config_mgr};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod notification_sys {
|
|
||||||
pub(crate) use rustfs_ecstore::notification_sys::get_global_notification_sys;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod pools {
|
|
||||||
pub(crate) use rustfs_ecstore::pools::{
|
|
||||||
PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod rio {
|
|
||||||
#[cfg(test)]
|
|
||||||
pub(crate) use rustfs_ecstore::rio::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader};
|
|
||||||
pub(crate) use rustfs_ecstore::rio::{
|
|
||||||
DynReader, HashReader, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod set_disk {
|
|
||||||
pub(crate) use rustfs_ecstore::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod store {
|
|
||||||
pub(crate) use rustfs_ecstore::store::ECStore;
|
|
||||||
#[cfg(test)]
|
|
||||||
pub(crate) use rustfs_ecstore::store::init_local_disks;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod tier {
|
|
||||||
pub(crate) use rustfs_ecstore::tier::tier;
|
|
||||||
#[cfg(test)]
|
|
||||||
pub(crate) use rustfs_ecstore::tier::{tier_config, warm_backend};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::storage_compat::ecstore::disk::DiskAPI;
|
use crate::storage_compat::DiskAPI;
|
||||||
use rustfs_io_metrics::capacity_metrics::{
|
use rustfs_io_metrics::capacity_metrics::{
|
||||||
record_capacity_cache_hit, record_capacity_cache_miss, record_capacity_cache_served, record_capacity_refresh_request,
|
record_capacity_cache_hit, record_capacity_cache_miss, record_capacity_cache_served, record_capacity_refresh_request,
|
||||||
record_capacity_scan_mode,
|
record_capacity_scan_mode,
|
||||||
@@ -263,7 +263,7 @@ pub async fn init_capacity_management_for_local_disks() {
|
|||||||
"Capacity manager state changed"
|
"Capacity manager state changed"
|
||||||
);
|
);
|
||||||
|
|
||||||
let disks = crate::storage_compat::ecstore::store::all_local_disk().await;
|
let disks = crate::storage_compat::all_local_disk().await;
|
||||||
if disks.is_empty() {
|
if disks.is_empty() {
|
||||||
warn!(
|
warn!(
|
||||||
component = LOG_COMPONENT_CAPACITY,
|
component = LOG_COMPONENT_CAPACITY,
|
||||||
|
|||||||
@@ -16,7 +16,7 @@
|
|||||||
#[allow(unsafe_op_in_unsafe_fn)]
|
#[allow(unsafe_op_in_unsafe_fn)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use crate::config::{CommandResult, Config, Opt, TlsCommands};
|
use crate::config::{CommandResult, Config, Opt, TlsCommands};
|
||||||
use crate::storage_compat::ecstore::disks_layout::DisksLayout;
|
use crate::storage_compat::DisksLayout;
|
||||||
use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_OBS_ENDPOINT, RUSTFS_REGION};
|
use rustfs_config::{DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_OBS_ENDPOINT, RUSTFS_REGION};
|
||||||
use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY};
|
use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY};
|
||||||
use serial_test::serial;
|
use serial_test::serial;
|
||||||
@@ -262,7 +262,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
#[serial]
|
||||||
fn test_volumes_and_disk_layout_parsing() {
|
fn test_volumes_and_disk_layout_parsing() {
|
||||||
use crate::storage_compat::ecstore::disks_layout::DisksLayout;
|
use crate::storage_compat::DisksLayout;
|
||||||
|
|
||||||
// Test case 1: Single volume path
|
// Test case 1: Single volume path
|
||||||
let args = vec!["rustfs", "/data/vol1"];
|
let args = vec!["rustfs", "/data/vol1"];
|
||||||
|
|||||||
+9
-18
@@ -51,20 +51,11 @@ use crate::init::{add_bucket_notification_configuration, init_buffer_profile_sys
|
|||||||
use crate::server::{ShutdownHandle, shutdown_event_notifier, start_http_server, stop_audit_system};
|
use crate::server::{ShutdownHandle, shutdown_event_notifier, start_http_server, stop_audit_system};
|
||||||
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
|
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
|
||||||
use crate::startup_iam::{bootstrap_or_defer_iam_init, publish_ready_for_iam_bootstrap};
|
use crate::startup_iam::{bootstrap_or_defer_iam_init, publish_ready_for_iam_bootstrap};
|
||||||
use crate::storage_compat::ecstore::store::init_lock_clients;
|
use crate::storage_compat::init_lock_clients;
|
||||||
use crate::storage_compat::ecstore::{
|
use crate::storage_compat::{
|
||||||
bucket::replication::init_background_replication,
|
ECStore, EndpointServerPools, init as init_ecstore_config, init_background_replication, init_bucket_metadata_sys,
|
||||||
bucket::{
|
init_global_config_sys, init_local_disks, set_global_endpoints, set_global_rustfs_port, try_migrate_bucket_metadata,
|
||||||
metadata_sys::init_bucket_metadata_sys,
|
try_migrate_iam_config, try_migrate_server_config, update_erasure_type,
|
||||||
migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
|
|
||||||
},
|
|
||||||
config as ecconfig,
|
|
||||||
endpoints::EndpointServerPools,
|
|
||||||
global::set_global_rustfs_port,
|
|
||||||
set_global_endpoints,
|
|
||||||
store::ECStore,
|
|
||||||
store::init_local_disks,
|
|
||||||
update_erasure_type,
|
|
||||||
};
|
};
|
||||||
use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
|
use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
|
||||||
use rustfs_credentials::init_global_action_credentials;
|
use rustfs_credentials::init_global_action_credentials;
|
||||||
@@ -333,7 +324,7 @@ impl RustFSServerBuilder {
|
|||||||
let region = region_str
|
let region = region_str
|
||||||
.parse()
|
.parse()
|
||||||
.map_err(|e| ServerError::Init(format!("invalid region '{region_str}': {e}")))?;
|
.map_err(|e| ServerError::Init(format!("invalid region '{region_str}': {e}")))?;
|
||||||
crate::storage_compat::ecstore::global::set_global_region(region);
|
crate::storage_compat::set_global_region(region);
|
||||||
}
|
}
|
||||||
|
|
||||||
let server_port = server_addr.port();
|
let server_port = server_addr.port();
|
||||||
@@ -389,12 +380,12 @@ impl RustFSServerBuilder {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
ecconfig::init();
|
init_ecstore_config();
|
||||||
ecconfig::try_migrate_server_config(store.clone()).await;
|
try_migrate_server_config(store.clone()).await;
|
||||||
|
|
||||||
// Global config system (with retry).
|
// Global config system (with retry).
|
||||||
let mut retry = 0;
|
let mut retry = 0;
|
||||||
while let Err(e) = ecconfig::init_global_config_sys(store.clone()).await {
|
while let Err(e) = init_global_config_sys(store.clone()).await {
|
||||||
retry += 1;
|
retry += 1;
|
||||||
if retry > 15 {
|
if retry > 15 {
|
||||||
shutdown_embedded_server();
|
shutdown_embedded_server();
|
||||||
|
|||||||
+2
-2
@@ -12,8 +12,8 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::storage_compat::ecstore::bucket::quota::QuotaError;
|
use crate::storage_compat::StorageError;
|
||||||
use crate::storage_compat::ecstore::error::StorageError;
|
use crate::storage_compat::quota::QuotaError;
|
||||||
use rustfs_storage_api::HTTPRangeError;
|
use rustfs_storage_api::HTTPRangeError;
|
||||||
use s3s::{S3Error, S3ErrorCode};
|
use s3s::{S3Error, S3ErrorCode};
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
use crate::server::ShutdownHandle;
|
use crate::server::ShutdownHandle;
|
||||||
use crate::storage::{process_lambda_configurations, process_queue_configurations, process_topic_configurations};
|
use crate::storage::{process_lambda_configurations, process_queue_configurations, process_topic_configurations};
|
||||||
use crate::storage_compat::ecstore::bucket::metadata_sys;
|
use crate::storage_compat::metadata_sys;
|
||||||
use crate::{admin, config, version};
|
use crate::{admin, config, version};
|
||||||
use rustfs_config::{
|
use rustfs_config::{
|
||||||
DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK,
|
DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK,
|
||||||
@@ -157,7 +157,7 @@ fn arn_to_target_id(arn_str: &str) -> Result<rustfs_targets::arn::TargetID, Targ
|
|||||||
/// * `buckets` - A vector of bucket names to process
|
/// * `buckets` - A vector of bucket names to process
|
||||||
#[instrument(skip_all)]
|
#[instrument(skip_all)]
|
||||||
pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
|
pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
|
||||||
let global_region = crate::storage_compat::ecstore::global::get_global_region();
|
let global_region = crate::storage_compat::get_global_region();
|
||||||
let region = global_region
|
let region = global_region
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.filter(|r| !r.as_str().is_empty())
|
.filter(|r| !r.as_str().is_empty())
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store};
|
use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store};
|
||||||
use crate::app::context::resolve_server_config;
|
use crate::app::context::resolve_server_config;
|
||||||
use crate::storage_compat::ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook};
|
use crate::storage_compat::{EcstoreEventArgs, register_event_dispatch_hook};
|
||||||
use rustfs_notify::EventArgs as NotifyEventArgs;
|
use rustfs_notify::EventArgs as NotifyEventArgs;
|
||||||
use rustfs_s3_types::EventName;
|
use rustfs_s3_types::EventName;
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ use crate::server::{
|
|||||||
use crate::storage;
|
use crate::storage;
|
||||||
use crate::storage::rpc::InternodeRpcService;
|
use crate::storage::rpc::InternodeRpcService;
|
||||||
use crate::storage::tonic_service::make_server;
|
use crate::storage::tonic_service::make_server;
|
||||||
use crate::storage_compat::ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
|
use crate::storage_compat::{TONIC_RPC_PREFIX, verify_rpc_signature};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use http::{HeaderMap, Method, Request as HttpRequest, Response};
|
use http::{HeaderMap, Method, Request as HttpRequest, Response};
|
||||||
use hyper_util::{
|
use hyper_util::{
|
||||||
|
|||||||
@@ -12,9 +12,9 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::storage_compat::ecstore::{
|
use crate::storage_compat::{
|
||||||
config::com::{read_config, save_config},
|
EcstoreError as StorageError,
|
||||||
error::Error as StorageError,
|
com::{read_config, save_config},
|
||||||
resolve_object_store_handle,
|
resolve_object_store_handle,
|
||||||
};
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|||||||
@@ -14,9 +14,9 @@
|
|||||||
|
|
||||||
use crate::server::{ServiceState, ServiceStateManager};
|
use crate::server::{ServiceState, ServiceStateManager};
|
||||||
use crate::server::{has_path_prefix, is_table_catalog_path};
|
use crate::server::{has_path_prefix, is_table_catalog_path};
|
||||||
use crate::storage_compat::ecstore::global::is_dist_erasure;
|
use crate::storage_compat::is_dist_erasure;
|
||||||
use crate::storage_compat::ecstore::global::{get_global_endpoints_opt, get_global_lock_clients};
|
use crate::storage_compat::resolve_object_store_handle;
|
||||||
use crate::storage_compat::ecstore::resolve_object_store_handle;
|
use crate::storage_compat::{get_global_endpoints_opt, get_global_lock_clients};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use http::{Request as HttpRequest, Response, StatusCode};
|
use http::{Request as HttpRequest, Response, StatusCode};
|
||||||
use http_body::Body;
|
use http_body::Body;
|
||||||
@@ -496,13 +496,10 @@ async fn collect_storage_readiness_uncached() -> bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn set_lock_quorum_status(
|
fn set_lock_quorum_status(online_hosts: &HashSet<String>, set_endpoints: &[crate::storage_compat::Endpoint]) -> LockQuorumStatus {
|
||||||
online_hosts: &HashSet<String>,
|
|
||||||
set_endpoints: &[crate::storage_compat::ecstore::disk::endpoint::Endpoint],
|
|
||||||
) -> LockQuorumStatus {
|
|
||||||
let total_clients = set_endpoints
|
let total_clients = set_endpoints
|
||||||
.iter()
|
.iter()
|
||||||
.map(crate::storage_compat::ecstore::disk::endpoint::Endpoint::host_port)
|
.map(crate::storage_compat::Endpoint::host_port)
|
||||||
.filter(|host| !host.is_empty())
|
.filter(|host| !host.is_empty())
|
||||||
.collect::<HashSet<_>>();
|
.collect::<HashSet<_>>();
|
||||||
let total_clients_len = total_clients.len();
|
let total_clients_len = total_clients.len();
|
||||||
@@ -526,7 +523,7 @@ fn set_lock_quorum_status(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn aggregate_lock_quorum_status(
|
fn aggregate_lock_quorum_status(
|
||||||
pool_endpoints: &crate::storage_compat::ecstore::endpoints::EndpointServerPools,
|
pool_endpoints: &crate::storage_compat::EndpointServerPools,
|
||||||
online_hosts: &HashSet<String>,
|
online_hosts: &HashSet<String>,
|
||||||
) -> LockQuorumStatus {
|
) -> LockQuorumStatus {
|
||||||
let mut connected_clients = 0usize;
|
let mut connected_clients = 0usize;
|
||||||
@@ -842,8 +839,8 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn aggregate_lock_quorum_status_requires_each_set_to_meet_quorum() {
|
fn aggregate_lock_quorum_status_requires_each_set_to_meet_quorum() {
|
||||||
use crate::storage_compat::ecstore::disk::endpoint::Endpoint;
|
use crate::storage_compat::Endpoint;
|
||||||
use crate::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
use crate::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||||
|
|
||||||
let endpoints = vec![
|
let endpoints = vec![
|
||||||
Endpoint {
|
Endpoint {
|
||||||
@@ -900,8 +897,8 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn aggregate_lock_quorum_status_fails_when_any_set_loses_quorum() {
|
fn aggregate_lock_quorum_status_fails_when_any_set_loses_quorum() {
|
||||||
use crate::storage_compat::ecstore::disk::endpoint::Endpoint;
|
use crate::storage_compat::Endpoint;
|
||||||
use crate::storage_compat::ecstore::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
use crate::storage_compat::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||||
|
|
||||||
let endpoints = vec![
|
let endpoints = vec![
|
||||||
Endpoint {
|
Endpoint {
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::storage_compat::ecstore::endpoints::EndpointServerPools;
|
use crate::storage_compat::EndpointServerPools;
|
||||||
use rustfs_config::{
|
use rustfs_config::{
|
||||||
DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, ENV_RUSTFS_UNSUPPORTED_FS_POLICY, RUSTFS_UNSUPPORTED_FS_POLICY_FAIL,
|
DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, ENV_RUSTFS_UNSUPPORTED_FS_POLICY, RUSTFS_UNSUPPORTED_FS_POLICY_FAIL,
|
||||||
RUSTFS_UNSUPPORTED_FS_POLICY_WARN,
|
RUSTFS_UNSUPPORTED_FS_POLICY_WARN,
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
use crate::app::context::{AppContext, get_global_app_context, init_global_app_context};
|
use crate::app::context::{AppContext, get_global_app_context, init_global_app_context};
|
||||||
use crate::server::{ServiceStateManager, publish_ready_when_runtime_ready};
|
use crate::server::{ServiceStateManager, publish_ready_when_runtime_ready};
|
||||||
use crate::storage_compat::ecstore::store::ECStore;
|
use crate::storage_compat::ECStore;
|
||||||
use rustfs_common::{GlobalReadiness, SystemStage};
|
use rustfs_common::{GlobalReadiness, SystemStage};
|
||||||
use rustfs_iam::init_iam_sys;
|
use rustfs_iam::init_iam_sys;
|
||||||
use rustfs_kms::KmsServiceManager;
|
use rustfs_kms::KmsServiceManager;
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::storage_compat::ecstore::global::set_global_rustfs_port;
|
use crate::storage_compat::set_global_rustfs_port;
|
||||||
use crate::{
|
use crate::{
|
||||||
capacity::capacity_integration::init_capacity_management,
|
capacity::capacity_integration::init_capacity_management,
|
||||||
config::Config,
|
config::Config,
|
||||||
@@ -57,7 +57,7 @@ pub async fn init_startup_listen_context(config: &Config) -> Result<StartupListe
|
|||||||
if let Some(region_str) = &config.region {
|
if let Some(region_str) = &config.region {
|
||||||
region_str
|
region_str
|
||||||
.parse::<s3s::region::Region>()
|
.parse::<s3s::region::Region>()
|
||||||
.map(crate::storage_compat::ecstore::global::set_global_region)
|
.map(crate::storage_compat::set_global_region)
|
||||||
.map_err(|err| Error::other(format!("invalid region '{}': {}", region_str, err)))?;
|
.map_err(|err| Error::other(format!("invalid region '{}': {}", region_str, err)))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,16 +12,9 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::storage_compat::ecstore::{
|
use crate::storage_compat::{
|
||||||
bucket::{
|
ECStore, EndpointServerPools, get_global_replication_pool, init_bucket_metadata_sys, new_global_notification_sys,
|
||||||
metadata_sys::init_bucket_metadata_sys,
|
shutdown_background_services, try_migrate_bucket_metadata, try_migrate_iam_config,
|
||||||
migration::{try_migrate_bucket_metadata, try_migrate_iam_config},
|
|
||||||
replication::get_global_replication_pool,
|
|
||||||
},
|
|
||||||
endpoints::EndpointServerPools,
|
|
||||||
global::shutdown_background_services,
|
|
||||||
notification_sys::new_global_notification_sys,
|
|
||||||
store::ECStore,
|
|
||||||
};
|
};
|
||||||
use crate::{
|
use crate::{
|
||||||
config::Config,
|
config::Config,
|
||||||
@@ -597,16 +590,14 @@ where
|
|||||||
start_audit().await
|
start_audit().await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn init_notification_system(endpoint_pools: EndpointServerPools) -> crate::storage_compat::ecstore::error::Result<()> {
|
pub async fn init_notification_system(endpoint_pools: EndpointServerPools) -> crate::storage_compat::EcstoreResult<()> {
|
||||||
init_notification_system_with(|| new_global_notification_sys(endpoint_pools)).await
|
init_notification_system_with(|| new_global_notification_sys(endpoint_pools)).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn init_notification_system_with<InitFn, InitFuture>(
|
async fn init_notification_system_with<InitFn, InitFuture>(init_notification: InitFn) -> crate::storage_compat::EcstoreResult<()>
|
||||||
init_notification: InitFn,
|
|
||||||
) -> crate::storage_compat::ecstore::error::Result<()>
|
|
||||||
where
|
where
|
||||||
InitFn: FnOnce() -> InitFuture,
|
InitFn: FnOnce() -> InitFuture,
|
||||||
InitFuture: Future<Output = crate::storage_compat::ecstore::error::Result<()>>,
|
InitFuture: Future<Output = crate::storage_compat::EcstoreResult<()>>,
|
||||||
{
|
{
|
||||||
init_notification().await
|
init_notification().await
|
||||||
}
|
}
|
||||||
@@ -664,8 +655,7 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn notification_system_returns_source_error() {
|
async fn notification_system_returns_source_error() {
|
||||||
let result =
|
let result = init_notification_system_with(|| async { Err(crate::storage_compat::EcstoreError::FaultyDisk) }).await;
|
||||||
init_notification_system_with(|| async { Err(crate::storage_compat::ecstore::error::Error::FaultyDisk) }).await;
|
|
||||||
|
|
||||||
assert!(result.is_err());
|
assert!(result.is_err());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,12 +13,9 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
|
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
|
||||||
use crate::storage_compat::ecstore::{
|
use crate::storage_compat::{
|
||||||
bucket::replication::init_background_replication,
|
ECStore, EndpointServerPools, init as init_ecstore_config, init_background_replication, init_global_config_sys,
|
||||||
config as ecconfig,
|
init_local_disks, init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints, try_migrate_server_config,
|
||||||
endpoints::EndpointServerPools,
|
|
||||||
set_global_endpoints,
|
|
||||||
store::{ECStore, init_local_disks, init_lock_clients, prewarm_local_disk_id_map},
|
|
||||||
update_erasure_type,
|
update_erasure_type,
|
||||||
};
|
};
|
||||||
use rustfs_common::{GlobalReadiness, SystemStage};
|
use rustfs_common::{GlobalReadiness, SystemStage};
|
||||||
@@ -147,11 +144,11 @@ pub async fn init_startup_storage_runtime(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn init_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> {
|
async fn init_startup_storage_global_config(store: Arc<ECStore>) -> Result<()> {
|
||||||
ecconfig::init();
|
init_ecstore_config();
|
||||||
ecconfig::try_migrate_server_config(store.clone()).await;
|
try_migrate_server_config(store.clone()).await;
|
||||||
|
|
||||||
let mut retry_count = 0;
|
let mut retry_count = 0;
|
||||||
while let Err(e) = ecconfig::init_global_config_sys(store.clone()).await {
|
while let Err(e) = init_global_config_sys(store.clone()).await {
|
||||||
let next_retry_count = retry_count + 1;
|
let next_retry_count = retry_count + 1;
|
||||||
error!(
|
error!(
|
||||||
target: "rustfs::main::run",
|
target: "rustfs::main::run",
|
||||||
|
|||||||
@@ -18,11 +18,11 @@ use crate::error::ApiError;
|
|||||||
use crate::license::license_check;
|
use crate::license::license_check;
|
||||||
use crate::server::RemoteAddr;
|
use crate::server::RemoteAddr;
|
||||||
use crate::storage::request_context::RequestContext;
|
use crate::storage::request_context::RequestContext;
|
||||||
use crate::storage::storage_compat::ecstore::bucket::metadata_sys;
|
use crate::storage::storage_compat::ECStore;
|
||||||
use crate::storage::storage_compat::ecstore::bucket::policy_sys::PolicySys;
|
use crate::storage::storage_compat::metadata_sys;
|
||||||
use crate::storage::storage_compat::ecstore::error::{StorageError, is_err_bucket_not_found};
|
use crate::storage::storage_compat::policy_sys::PolicySys;
|
||||||
use crate::storage::storage_compat::ecstore::resolve_object_store_handle;
|
use crate::storage::storage_compat::resolve_object_store_handle;
|
||||||
use crate::storage::storage_compat::ecstore::store::ECStore;
|
use crate::storage::storage_compat::{StorageError, is_err_bucket_not_found};
|
||||||
use metrics::counter;
|
use metrics::counter;
|
||||||
use rustfs_iam::error::Error as IamError;
|
use rustfs_iam::error::Error as IamError;
|
||||||
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
|
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
|
||||||
@@ -930,7 +930,7 @@ impl S3Access for FS {
|
|||||||
let req_info = ReqInfo {
|
let req_info = ReqInfo {
|
||||||
cred,
|
cred,
|
||||||
is_owner,
|
is_owner,
|
||||||
region: crate::storage::storage_compat::ecstore::global::get_global_region(),
|
region: crate::storage::storage_compat::get_global_region(),
|
||||||
request_context,
|
request_context,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|||||||
+12
-14
@@ -21,21 +21,19 @@ use crate::storage::access::has_bypass_governance_header;
|
|||||||
use crate::storage::helper::OperationHelper;
|
use crate::storage::helper::OperationHelper;
|
||||||
use crate::storage::options::get_opts;
|
use crate::storage::options::get_opts;
|
||||||
use crate::storage::s3_api::acl;
|
use crate::storage::s3_api::acl;
|
||||||
use crate::storage::storage_compat::ecstore::{
|
use crate::storage::storage_compat::{
|
||||||
bucket::{
|
StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||||
metadata::{
|
metadata::{
|
||||||
BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG,
|
BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_VERSIONING_CONFIG,
|
||||||
BUCKET_WEBSITE_CONFIG, OBJECT_LOCK_CONFIG,
|
BUCKET_WEBSITE_CONFIG, OBJECT_LOCK_CONFIG,
|
||||||
},
|
|
||||||
metadata_sys,
|
|
||||||
object_lock::objectlock_sys::check_retention_for_modification,
|
|
||||||
replication::{GLOBAL_REPLICATION_STATS, ReplicationConfigurationExt},
|
|
||||||
tagging::{decode_tags, decode_tags_to_map, encode_tags},
|
|
||||||
utils::serialize,
|
|
||||||
versioning::VersioningApi,
|
|
||||||
versioning_sys::BucketVersioningSys,
|
|
||||||
},
|
},
|
||||||
error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found},
|
metadata_sys,
|
||||||
|
object_lock::objectlock_sys::check_retention_for_modification,
|
||||||
|
replication::{GLOBAL_REPLICATION_STATS, ReplicationConfigurationExt},
|
||||||
|
tagging::{decode_tags, decode_tags_to_map, encode_tags},
|
||||||
|
utils::serialize,
|
||||||
|
versioning::VersioningApi,
|
||||||
|
versioning_sys::BucketVersioningSys,
|
||||||
};
|
};
|
||||||
use crate::storage::{parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled};
|
use crate::storage::{parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled};
|
||||||
use crate::table_catalog;
|
use crate::table_catalog;
|
||||||
|
|||||||
@@ -16,12 +16,12 @@ use crate::config::{RustFSBufferConfig, WorkloadProfile, get_global_buffer_confi
|
|||||||
use crate::error::ApiError;
|
use crate::error::ApiError;
|
||||||
use crate::server::cors;
|
use crate::server::cors;
|
||||||
use crate::storage::ecfs::ListObjectUnorderedQuery;
|
use crate::storage::ecfs::ListObjectUnorderedQuery;
|
||||||
use crate::storage::storage_compat::ecstore::bucket::metadata_sys;
|
use crate::storage::storage_compat::StorageError;
|
||||||
use crate::storage::storage_compat::ecstore::bucket::metadata_sys::get_replication_config;
|
use crate::storage::storage_compat::metadata_sys;
|
||||||
use crate::storage::storage_compat::ecstore::bucket::object_lock::objectlock_sys;
|
use crate::storage::storage_compat::metadata_sys::get_replication_config;
|
||||||
use crate::storage::storage_compat::ecstore::bucket::replication::ReplicationConfigurationExt;
|
use crate::storage::storage_compat::object_lock::objectlock_sys;
|
||||||
use crate::storage::storage_compat::ecstore::error::StorageError;
|
use crate::storage::storage_compat::replication::ReplicationConfigurationExt;
|
||||||
use crate::storage::storage_compat::ecstore::resolve_object_store_handle;
|
use crate::storage::storage_compat::resolve_object_store_handle;
|
||||||
use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE};
|
use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE};
|
||||||
use http::{HeaderMap, HeaderValue, StatusCode};
|
use http::{HeaderMap, HeaderValue, StatusCode};
|
||||||
use metrics::counter;
|
use metrics::counter;
|
||||||
@@ -739,7 +739,7 @@ pub(crate) async fn has_replication_rules(bucket: &str, objects: &[ObjectToDelet
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Helper function to get store and validate bucket exists
|
/// Helper function to get store and validate bucket exists
|
||||||
pub(crate) async fn get_validated_store(bucket: &str) -> S3Result<Arc<crate::storage::storage_compat::ecstore::store::ECStore>> {
|
pub(crate) async fn get_validated_store(bucket: &str) -> S3Result<Arc<crate::storage::storage_compat::ECStore>> {
|
||||||
let Some(store) = resolve_object_store_handle() else {
|
let Some(store) = resolve_object_store_handle() else {
|
||||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -18,8 +18,8 @@ mod tests {
|
|||||||
use crate::server::cors;
|
use crate::server::cors;
|
||||||
use crate::storage::ecfs::{FS, validate_object_lock_configuration_input};
|
use crate::storage::ecfs::{FS, validate_object_lock_configuration_input};
|
||||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||||
use crate::storage::storage_compat::ecstore::bucket::{metadata::BucketMetadata, metadata_sys};
|
use crate::storage::storage_compat::DEFAULT_READ_BUFFER_SIZE;
|
||||||
use crate::storage::storage_compat::ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
use crate::storage::storage_compat::{metadata::BucketMetadata, metadata_sys};
|
||||||
use crate::storage::{
|
use crate::storage::{
|
||||||
StorageObjectInfo as ObjectInfo, apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions,
|
StorageObjectInfo as ObjectInfo, apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions,
|
||||||
get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal, matches_origin_pattern, parse_etag,
|
get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal, matches_origin_pattern, parse_etag,
|
||||||
@@ -940,12 +940,12 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_validate_bucket_object_lock_enabled() {
|
async fn test_validate_bucket_object_lock_enabled() {
|
||||||
use crate::storage::storage_compat::ecstore::bucket::metadata::BucketMetadata;
|
use crate::storage::storage_compat::metadata::BucketMetadata;
|
||||||
use crate::storage::storage_compat::ecstore::bucket::metadata_sys::set_bucket_metadata;
|
use crate::storage::storage_compat::metadata_sys::set_bucket_metadata;
|
||||||
use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
|
use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
|
|
||||||
if crate::storage::storage_compat::ecstore::bucket::metadata_sys::GLOBAL_BucketMetadataSys
|
if crate::storage::storage_compat::metadata_sys::GLOBAL_BucketMetadataSys
|
||||||
.get()
|
.get()
|
||||||
.is_none()
|
.is_none()
|
||||||
{
|
{
|
||||||
@@ -1783,7 +1783,7 @@ mod tests {
|
|||||||
/// with a single-element vec value, matching the format expected by policy evaluation.
|
/// with a single-element vec value, matching the format expected by policy evaluation.
|
||||||
#[test]
|
#[test]
|
||||||
fn test_object_tag_condition_key_format() {
|
fn test_object_tag_condition_key_format() {
|
||||||
use crate::storage::storage_compat::ecstore::bucket::tagging::decode_tags_to_map;
|
use crate::storage::storage_compat::tagging::decode_tags_to_map;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
|
||||||
let tags_str = "security=public&project=webapp&env=prod";
|
let tags_str = "security=public&project=webapp&env=prod";
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::storage::storage_compat::ecstore::store::ECStore;
|
use crate::storage::storage_compat::ECStore;
|
||||||
use rustfs_storage_api::ListOperations as _;
|
use rustfs_storage_api::ListOperations as _;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
|||||||
@@ -29,11 +29,11 @@ pub mod timeout_wrapper;
|
|||||||
pub mod tonic_service;
|
pub mod tonic_service;
|
||||||
|
|
||||||
pub(crate) type StorageDeletedObject = rustfs_storage_api::DeletedObject;
|
pub(crate) type StorageDeletedObject = rustfs_storage_api::DeletedObject;
|
||||||
pub(crate) type StorageGetObjectReader = crate::storage::storage_compat::ecstore::GetObjectReader;
|
pub(crate) type StorageGetObjectReader = crate::storage::storage_compat::GetObjectReader;
|
||||||
pub(crate) type StorageObjectInfo = crate::storage::storage_compat::ecstore::ObjectInfo;
|
pub(crate) type StorageObjectInfo = crate::storage::storage_compat::ObjectInfo;
|
||||||
pub(crate) type StorageObjectOptions = crate::storage::storage_compat::ecstore::ObjectOptions;
|
pub(crate) type StorageObjectOptions = crate::storage::storage_compat::ObjectOptions;
|
||||||
pub(crate) type StorageObjectToDelete = rustfs_storage_api::ObjectToDelete;
|
pub(crate) type StorageObjectToDelete = rustfs_storage_api::ObjectToDelete;
|
||||||
pub(crate) type StoragePutObjReader = crate::storage::storage_compat::ecstore::PutObjReader;
|
pub(crate) type StoragePutObjReader = crate::storage::storage_compat::PutObjReader;
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod concurrent_fix_test;
|
mod concurrent_fix_test;
|
||||||
|
|||||||
@@ -12,9 +12,9 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::storage::storage_compat::ecstore::bucket::versioning_sys::BucketVersioningSys;
|
use crate::storage::storage_compat::Result;
|
||||||
use crate::storage::storage_compat::ecstore::error::Result;
|
use crate::storage::storage_compat::StorageError;
|
||||||
use crate::storage::storage_compat::ecstore::error::StorageError;
|
use crate::storage::storage_compat::versioning_sys::BucketVersioningSys;
|
||||||
use http::header::{IF_MATCH, IF_NONE_MATCH};
|
use http::header::{IF_MATCH, IF_NONE_MATCH};
|
||||||
use http::{HeaderMap, HeaderValue};
|
use http::{HeaderMap, HeaderValue};
|
||||||
use rustfs_utils::http::{
|
use rustfs_utils::http::{
|
||||||
|
|||||||
@@ -14,10 +14,10 @@
|
|||||||
|
|
||||||
use crate::server::RPC_PREFIX;
|
use crate::server::RPC_PREFIX;
|
||||||
use crate::storage::request_context::spawn_traced;
|
use crate::storage::request_context::spawn_traced;
|
||||||
use crate::storage::storage_compat::ecstore::disk::{DiskAPI, WalkDirOptions};
|
use crate::storage::storage_compat::DEFAULT_READ_BUFFER_SIZE;
|
||||||
use crate::storage::storage_compat::ecstore::rpc::verify_rpc_signature;
|
use crate::storage::storage_compat::find_local_disk_by_ref;
|
||||||
use crate::storage::storage_compat::ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
use crate::storage::storage_compat::verify_rpc_signature;
|
||||||
use crate::storage::storage_compat::ecstore::store::find_local_disk_by_ref;
|
use crate::storage::storage_compat::{DiskAPI, WalkDirOptions};
|
||||||
use bytes::{Bytes, BytesMut};
|
use bytes::{Bytes, BytesMut};
|
||||||
use futures_util::TryStreamExt;
|
use futures_util::TryStreamExt;
|
||||||
use http::{HeaderMap, Method, Request, Response, StatusCode, Uri};
|
use http::{HeaderMap, Method, Request, Response, StatusCode, Uri};
|
||||||
|
|||||||
@@ -16,22 +16,12 @@ use crate::admin::service::{
|
|||||||
config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot},
|
config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot},
|
||||||
site_replication::reload_site_replication_runtime_state,
|
site_replication::reload_site_replication_runtime_state,
|
||||||
};
|
};
|
||||||
use crate::storage::storage_compat::ecstore::{
|
use crate::storage::storage_compat::{
|
||||||
admin_server_info::get_local_server_property,
|
CollectMetricsOpts, DeleteOptions, DiskAPI, DiskError, DiskInfoOptions, DiskStore, FileInfoVersions, GLOBAL_TierConfigMgr,
|
||||||
bucket::{metadata::load_bucket_metadata, metadata_sys},
|
LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, ReadMultipleReq, ReadMultipleResp,
|
||||||
disk::{
|
ReadOptions, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, UpdateMetadataOpts, all_local_disk_path,
|
||||||
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
collect_local_metrics, find_local_disk_by_ref, get_global_lock_client, get_local_server_property,
|
||||||
UpdateMetadataOpts, error::DiskError,
|
metadata::load_bucket_metadata, metadata_sys, resolve_object_store_handle,
|
||||||
},
|
|
||||||
get_global_lock_client,
|
|
||||||
global::GLOBAL_TierConfigMgr,
|
|
||||||
metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics},
|
|
||||||
resolve_object_store_handle,
|
|
||||||
rpc::{
|
|
||||||
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
|
|
||||||
SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
|
||||||
},
|
|
||||||
store::{all_local_disk_path, find_local_disk_by_ref},
|
|
||||||
};
|
};
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use futures::Stream;
|
use futures::Stream;
|
||||||
@@ -132,9 +122,7 @@ fn unimplemented_rpc(method: &str) -> Status {
|
|||||||
Status::unimplemented(format!("{method} is not implemented"))
|
Status::unimplemented(format!("{method} is not implemented"))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn background_rebalance_start_error_message(
|
fn background_rebalance_start_error_message(result: crate::storage::storage_compat::Result<()>) -> Option<String> {
|
||||||
result: crate::storage::storage_compat::ecstore::error::Result<()>,
|
|
||||||
) -> Option<String> {
|
|
||||||
result.err().map(|err| format!("start_rebalance failed: {err}"))
|
result.err().map(|err| format!("start_rebalance failed: {err}"))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2378,9 +2366,8 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_background_rebalance_start_error_message_formats_error() {
|
fn test_background_rebalance_start_error_message_formats_error() {
|
||||||
let message =
|
let message = background_rebalance_start_error_message(Err(crate::storage::storage_compat::Error::other("boom")))
|
||||||
background_rebalance_start_error_message(Err(crate::storage::storage_compat::ecstore::error::Error::other("boom")))
|
.expect("background rebalance start failure should be formatted");
|
||||||
.expect("background rebalance start failure should be formatted");
|
|
||||||
|
|
||||||
assert!(message.contains("start_rebalance failed"));
|
assert!(message.contains("start_rebalance failed"));
|
||||||
assert!(message.contains("boom"));
|
assert!(message.contains("boom"));
|
||||||
@@ -2710,7 +2697,7 @@ mod tests {
|
|||||||
vars.insert(PEER_RESTSIGNAL.to_string(), SERVICE_SIGNAL_RELOAD_DYNAMIC.to_string());
|
vars.insert(PEER_RESTSIGNAL.to_string(), SERVICE_SIGNAL_RELOAD_DYNAMIC.to_string());
|
||||||
vars.insert(
|
vars.insert(
|
||||||
PEER_RESTSUB_SYS.to_string(),
|
PEER_RESTSUB_SYS.to_string(),
|
||||||
crate::storage::storage_compat::ecstore::config::com::STORAGE_CLASS_SUB_SYS.to_string(),
|
crate::storage::storage_compat::com::STORAGE_CLASS_SUB_SYS.to_string(),
|
||||||
);
|
);
|
||||||
|
|
||||||
let request = Request::new(SignalServiceRequest {
|
let request = Request::new(SignalServiceRequest {
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::storage::s3_api::common::rustfs_owner;
|
use crate::storage::s3_api::common::rustfs_owner;
|
||||||
use crate::storage::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
|
use crate::storage::storage_compat::object_api_utils::to_s3s_etag;
|
||||||
use percent_encoding::percent_decode_str;
|
use percent_encoding::percent_decode_str;
|
||||||
use rustfs_storage_api::{
|
use rustfs_storage_api::{
|
||||||
BucketInfo, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
BucketInfo, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||||
use crate::storage::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
|
use crate::storage::storage_compat::object_api_utils::to_s3s_etag;
|
||||||
use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo};
|
use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo};
|
||||||
use s3s::dto::{CommonPrefix, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp};
|
use s3s::dto::{CommonPrefix, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp};
|
||||||
use s3s::{S3Error, S3ErrorCode};
|
use s3s::{S3Error, S3ErrorCode};
|
||||||
@@ -191,7 +191,7 @@ mod tests {
|
|||||||
parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
|
parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
|
||||||
};
|
};
|
||||||
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
|
||||||
use crate::storage::storage_compat::ecstore::client::object_api_utils::to_s3s_etag;
|
use crate::storage::storage_compat::object_api_utils::to_s3s_etag;
|
||||||
use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo};
|
use rustfs_storage_api::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo};
|
||||||
use s3s::S3ErrorCode;
|
use s3s::S3ErrorCode;
|
||||||
use s3s::dto::Timestamp;
|
use s3s::dto::Timestamp;
|
||||||
|
|||||||
@@ -69,7 +69,7 @@
|
|||||||
//! }
|
//! }
|
||||||
//! ```
|
//! ```
|
||||||
|
|
||||||
use crate::storage::storage_compat::ecstore::error::StorageError;
|
use crate::storage::storage_compat::StorageError;
|
||||||
use aes_gcm::{
|
use aes_gcm::{
|
||||||
Aes256Gcm, Key, Nonce,
|
Aes256Gcm, Key, Nonce,
|
||||||
aead::{Aead, KeyInit},
|
aead::{Aead, KeyInit},
|
||||||
@@ -128,8 +128,8 @@ const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE;
|
|||||||
const OBJECT_KEY_DERIVATION_CONTEXT: &[u8] = b"object-encryption-key generation";
|
const OBJECT_KEY_DERIVATION_CONTEXT: &[u8] = b"object-encryption-key generation";
|
||||||
|
|
||||||
use crate::error::ApiError;
|
use crate::error::ApiError;
|
||||||
use crate::storage::storage_compat::ecstore::bucket::metadata_sys;
|
use crate::storage::storage_compat::Error;
|
||||||
use crate::storage::storage_compat::ecstore::error::Error;
|
use crate::storage::storage_compat::metadata_sys;
|
||||||
use rustfs_utils::http::headers::{
|
use rustfs_utils::http::headers::{
|
||||||
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
|
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY,
|
||||||
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT,
|
AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT,
|
||||||
@@ -745,29 +745,19 @@ pub struct ManagedSealedKey {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl EncryptionMaterial {
|
impl EncryptionMaterial {
|
||||||
pub fn write_encryption(
|
pub fn write_encryption(&self, multipart_part_number: Option<usize>) -> crate::storage::storage_compat::WriteEncryption {
|
||||||
&self,
|
|
||||||
multipart_part_number: Option<usize>,
|
|
||||||
) -> crate::storage::storage_compat::ecstore::rio::WriteEncryption {
|
|
||||||
match (self.key_kind, multipart_part_number) {
|
match (self.key_kind, multipart_part_number) {
|
||||||
(EncryptionKeyKind::Object, Some(part_number)) => {
|
(EncryptionKeyKind::Object, Some(part_number)) => {
|
||||||
crate::storage::storage_compat::ecstore::rio::WriteEncryption::multipart_object_key(
|
crate::storage::storage_compat::WriteEncryption::multipart_object_key(self.key_bytes, part_number as u32)
|
||||||
self.key_bytes,
|
|
||||||
part_number as u32,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
(EncryptionKeyKind::Object, None) => {
|
(EncryptionKeyKind::Object, None) => {
|
||||||
crate::storage::storage_compat::ecstore::rio::WriteEncryption::singlepart_object_key(self.key_bytes)
|
crate::storage::storage_compat::WriteEncryption::singlepart_object_key(self.key_bytes)
|
||||||
}
|
}
|
||||||
(EncryptionKeyKind::Direct, Some(part_number)) => {
|
(EncryptionKeyKind::Direct, Some(part_number)) => {
|
||||||
crate::storage::storage_compat::ecstore::rio::WriteEncryption::multipart(
|
crate::storage::storage_compat::WriteEncryption::multipart(self.key_bytes, self.base_nonce, part_number)
|
||||||
self.key_bytes,
|
|
||||||
self.base_nonce,
|
|
||||||
part_number,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
(EncryptionKeyKind::Direct, None) => {
|
(EncryptionKeyKind::Direct, None) => {
|
||||||
crate::storage::storage_compat::ecstore::rio::WriteEncryption::singlepart(self.key_bytes, self.base_nonce)
|
crate::storage::storage_compat::WriteEncryption::singlepart(self.key_bytes, self.base_nonce)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,70 +12,34 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) use rustfs_ecstore::admin_server_info::get_local_server_property;
|
||||||
pub(crate) use rustfs_ecstore::global::{get_global_lock_client, resolve_object_store_handle};
|
pub(crate) use rustfs_ecstore::bucket::{
|
||||||
|
metadata, metadata_sys, object_lock, policy_sys, replication, tagging, utils, versioning, versioning_sys,
|
||||||
|
};
|
||||||
|
pub(crate) use rustfs_ecstore::client::object_api_utils;
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) use rustfs_ecstore::config::com;
|
||||||
|
pub(crate) use rustfs_ecstore::disk::error::DiskError;
|
||||||
|
pub(crate) use rustfs_ecstore::disk::{
|
||||||
|
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
||||||
|
UpdateMetadataOpts, WalkDirOptions,
|
||||||
|
};
|
||||||
|
pub(crate) use rustfs_ecstore::error::{
|
||||||
|
Error, Result, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||||
|
};
|
||||||
|
pub(crate) use rustfs_ecstore::global::{
|
||||||
|
GLOBAL_TierConfigMgr, get_global_lock_client, get_global_region, resolve_object_store_handle,
|
||||||
|
};
|
||||||
|
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
||||||
|
pub(crate) use rustfs_ecstore::rio::WriteEncryption;
|
||||||
|
pub(crate) use rustfs_ecstore::rpc::{
|
||||||
|
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||||
|
SERVICE_SIGNAL_RELOAD_DYNAMIC, verify_rpc_signature,
|
||||||
|
};
|
||||||
|
pub(crate) use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
||||||
|
pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk_path, find_local_disk_by_ref};
|
||||||
|
|
||||||
pub(crate) mod admin_server_info {
|
pub(crate) type GetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
|
||||||
pub(crate) use rustfs_ecstore::admin_server_info::get_local_server_property;
|
pub(crate) type ObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
||||||
}
|
pub(crate) type ObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
|
||||||
|
pub(crate) type PutObjReader = rustfs_ecstore::store_api::PutObjReader;
|
||||||
pub(crate) mod bucket {
|
|
||||||
pub(crate) use rustfs_ecstore::bucket::{
|
|
||||||
metadata, metadata_sys, object_lock, policy_sys, replication, tagging, utils, versioning, versioning_sys,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod client {
|
|
||||||
pub(crate) use rustfs_ecstore::client::object_api_utils;
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
pub(crate) mod config {
|
|
||||||
pub(crate) use rustfs_ecstore::config::com;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod disk {
|
|
||||||
pub(crate) use rustfs_ecstore::disk::{
|
|
||||||
DeleteOptions, DiskAPI, DiskInfoOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
|
||||||
UpdateMetadataOpts, WalkDirOptions, error,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod error {
|
|
||||||
pub(crate) use rustfs_ecstore::error::{
|
|
||||||
Error, Result, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod global {
|
|
||||||
pub(crate) use rustfs_ecstore::global::{GLOBAL_TierConfigMgr, get_global_region};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod metrics_realtime {
|
|
||||||
pub(crate) use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod rio {
|
|
||||||
pub(crate) use rustfs_ecstore::rio::WriteEncryption;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod rpc {
|
|
||||||
pub(crate) use rustfs_ecstore::rpc::{
|
|
||||||
LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerS3Client, SERVICE_SIGNAL_REFRESH_CONFIG,
|
|
||||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, verify_rpc_signature,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod set_disk {
|
|
||||||
pub(crate) use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod store {
|
|
||||||
pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk_path, find_local_disk_by_ref};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) type GetObjectReader = rustfs_ecstore::store_api::GetObjectReader;
|
|
||||||
pub(crate) type ObjectInfo = rustfs_ecstore::store_api::ObjectInfo;
|
|
||||||
pub(crate) type ObjectOptions = rustfs_ecstore::store_api::ObjectOptions;
|
|
||||||
pub(crate) type PutObjReader = rustfs_ecstore::store_api::PutObjReader;
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -12,62 +12,24 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys;
|
||||||
pub(crate) use rustfs_ecstore::global::{resolve_object_store_handle, set_global_endpoints, update_erasure_type};
|
pub(crate) use rustfs_ecstore::bucket::migration::{try_migrate_bucket_metadata, try_migrate_iam_config};
|
||||||
|
pub(crate) use rustfs_ecstore::bucket::replication::{get_global_replication_pool, init_background_replication};
|
||||||
pub(crate) mod bucket {
|
pub(crate) use rustfs_ecstore::bucket::{metadata, metadata_sys, quota};
|
||||||
pub(crate) use rustfs_ecstore::bucket::{metadata, metadata_sys, migration, quota, replication};
|
pub(crate) use rustfs_ecstore::config::{com, init, init_global_config_sys, try_migrate_server_config};
|
||||||
}
|
pub(crate) use rustfs_ecstore::disk::{DiskAPI, RUSTFS_META_BUCKET, endpoint::Endpoint};
|
||||||
|
#[cfg(test)]
|
||||||
pub(crate) mod config {
|
pub(crate) use rustfs_ecstore::disks_layout::DisksLayout;
|
||||||
pub(crate) use rustfs_ecstore::config::{com, init, init_global_config_sys, try_migrate_server_config};
|
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
|
||||||
}
|
#[cfg(test)]
|
||||||
|
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
|
||||||
pub(crate) mod disk {
|
pub(crate) use rustfs_ecstore::error::{Error as EcstoreError, Result as EcstoreResult, StorageError};
|
||||||
pub(crate) use rustfs_ecstore::disk::{DiskAPI, RUSTFS_META_BUCKET, endpoint};
|
pub(crate) use rustfs_ecstore::event_notification::{EventArgs as EcstoreEventArgs, register_event_dispatch_hook};
|
||||||
}
|
pub(crate) use rustfs_ecstore::global::{
|
||||||
|
get_global_endpoints_opt, get_global_lock_clients, get_global_region, is_dist_erasure, resolve_object_store_handle,
|
||||||
#[cfg(test)]
|
set_global_endpoints, set_global_region, set_global_rustfs_port, shutdown_background_services, update_erasure_type,
|
||||||
pub(crate) mod disks_layout {
|
};
|
||||||
pub(crate) use rustfs_ecstore::disks_layout::DisksLayout;
|
pub(crate) use rustfs_ecstore::notification_sys::new_global_notification_sys;
|
||||||
}
|
pub(crate) use rustfs_ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
|
||||||
|
pub(crate) use rustfs_ecstore::set_disk::get_lock_acquire_timeout;
|
||||||
pub(crate) mod endpoints {
|
pub(crate) use rustfs_ecstore::store::{ECStore, all_local_disk, init_local_disks, init_lock_clients, prewarm_local_disk_id_map};
|
||||||
pub(crate) use rustfs_ecstore::endpoints::EndpointServerPools;
|
|
||||||
#[cfg(test)]
|
|
||||||
pub(crate) use rustfs_ecstore::endpoints::{Endpoints, PoolEndpoints};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod error {
|
|
||||||
pub(crate) use rustfs_ecstore::error::{Error, Result, StorageError};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod event_notification {
|
|
||||||
pub(crate) use rustfs_ecstore::event_notification::{EventArgs, register_event_dispatch_hook};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod global {
|
|
||||||
pub(crate) use rustfs_ecstore::global::{
|
|
||||||
get_global_endpoints_opt, get_global_lock_clients, get_global_region, is_dist_erasure, set_global_region,
|
|
||||||
set_global_rustfs_port, shutdown_background_services,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod notification_sys {
|
|
||||||
pub(crate) use rustfs_ecstore::notification_sys::new_global_notification_sys;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod rpc {
|
|
||||||
pub(crate) use rustfs_ecstore::rpc::{TONIC_RPC_PREFIX, verify_rpc_signature};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod set_disk {
|
|
||||||
pub(crate) use rustfs_ecstore::set_disk::get_lock_acquire_timeout;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) mod store {
|
|
||||||
pub(crate) use rustfs_ecstore::store::{
|
|
||||||
ECStore, all_local_disk, init_local_disks, init_lock_clients, prewarm_local_disk_id_map,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
+12
-12
@@ -27,16 +27,16 @@ use std::{
|
|||||||
time::{Duration as StdDuration, Instant},
|
time::{Duration as StdDuration, Instant},
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::storage_compat::ecstore::bucket::{
|
use crate::storage_compat::RUSTFS_META_BUCKET;
|
||||||
|
use crate::storage_compat::get_lock_acquire_timeout;
|
||||||
|
use crate::storage_compat::{EcstoreError, StorageError};
|
||||||
|
use crate::storage_compat::{
|
||||||
metadata::{
|
metadata::{
|
||||||
BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG,
|
BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG,
|
||||||
BUCKET_TABLE_RESERVED_PREFIX, table_catalog_path_hash,
|
BUCKET_TABLE_RESERVED_PREFIX, table_catalog_path_hash,
|
||||||
},
|
},
|
||||||
metadata_sys,
|
metadata_sys,
|
||||||
};
|
};
|
||||||
use crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET;
|
|
||||||
use crate::storage_compat::ecstore::error::{Error as EcstoreError, StorageError};
|
|
||||||
use crate::storage_compat::ecstore::set_disk::get_lock_acquire_timeout;
|
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use datafusion::{
|
use datafusion::{
|
||||||
arrow::datatypes::SchemaRef,
|
arrow::datatypes::SchemaRef,
|
||||||
@@ -7813,7 +7813,7 @@ mod tests {
|
|||||||
.map(|(bucket, _)| bucket.as_str())
|
.map(|(bucket, _)| bucket.as_str())
|
||||||
.collect::<BTreeSet<_>>();
|
.collect::<BTreeSet<_>>();
|
||||||
|
|
||||||
assert_eq!(object_buckets, BTreeSet::from([crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET]));
|
assert_eq!(object_buckets, BTreeSet::from([crate::storage_compat::RUSTFS_META_BUCKET]));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -10777,7 +10777,7 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||||
backend
|
backend
|
||||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &idempotency_path, 1)
|
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &idempotency_path, 1)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
let err = store
|
let err = store
|
||||||
@@ -10827,7 +10827,7 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||||
backend
|
backend
|
||||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
|
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
let request = TableCommitRequest {
|
let request = TableCommitRequest {
|
||||||
@@ -10882,7 +10882,7 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||||
backend
|
backend
|
||||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
|
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
let result = store
|
let result = store
|
||||||
@@ -10936,7 +10936,7 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||||
backend
|
backend
|
||||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
|
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
store
|
store
|
||||||
@@ -10990,7 +10990,7 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||||
backend
|
backend
|
||||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &commit_path, 2)
|
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &commit_path, 2)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
store
|
store
|
||||||
@@ -11041,7 +11041,7 @@ mod tests {
|
|||||||
backend.seed_object(bucket, ¤t_metadata, b"{}".to_vec()).await;
|
backend.seed_object(bucket, ¤t_metadata, b"{}".to_vec()).await;
|
||||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||||
backend
|
backend
|
||||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &table_path, 2)
|
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &table_path, 2)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
let err = store
|
let err = store
|
||||||
@@ -11098,7 +11098,7 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
backend.seed_object(bucket, &new_metadata, b"{}".to_vec()).await;
|
||||||
backend
|
backend
|
||||||
.fail_put_attempt(crate::storage_compat::ecstore::disk::RUSTFS_META_BUCKET, &idempotency_path, 2)
|
.fail_put_attempt(crate::storage_compat::RUSTFS_META_BUCKET, &idempotency_path, 2)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
let result = store
|
let result = store
|
||||||
|
|||||||
@@ -65,6 +65,10 @@ STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_list_c
|
|||||||
STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_operation_consumer_hits.txt"
|
STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_operation_consumer_hits.txt"
|
||||||
STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE="${TMP_DIR}/store_api_object_operation_local_method_hits.txt"
|
STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE="${TMP_DIR}/store_api_object_operation_local_method_hits.txt"
|
||||||
DIRECT_ECSTORE_IMPORT_HITS_FILE="${TMP_DIR}/direct_ecstore_import_hits.txt"
|
DIRECT_ECSTORE_IMPORT_HITS_FILE="${TMP_DIR}/direct_ecstore_import_hits.txt"
|
||||||
|
TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/test_harness_nested_storage_compat_hits.txt"
|
||||||
|
RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/rustfs_nested_storage_compat_hits.txt"
|
||||||
|
RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/rustfs_runtime_scalar_storage_compat_hits.txt"
|
||||||
|
RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE="${TMP_DIR}/rustfs_runtime_secondary_storage_compat_hits.txt"
|
||||||
PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE="${TMP_DIR}/production_unused_compat_allow_hits.txt"
|
PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE="${TMP_DIR}/production_unused_compat_allow_hits.txt"
|
||||||
BROAD_STORE_API_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/broad_store_api_compat_reexport_hits.txt"
|
BROAD_STORE_API_COMPAT_REEXPORT_HITS_FILE="${TMP_DIR}/broad_store_api_compat_reexport_hits.txt"
|
||||||
NESTED_STORE_API_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/nested_store_api_compat_module_hits.txt"
|
NESTED_STORE_API_COMPAT_MODULE_HITS_FILE="${TMP_DIR}/nested_store_api_compat_module_hits.txt"
|
||||||
@@ -204,10 +208,18 @@ require_source_line \
|
|||||||
"crates/storage-api/src/lib.rs" \
|
"crates/storage-api/src/lib.rs" \
|
||||||
"pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};" \
|
"pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};" \
|
||||||
"storage-api public bucket contract re-export"
|
"storage-api public bucket contract re-export"
|
||||||
|
require_source_line \
|
||||||
|
"crates/storage-api/src/lib.rs" \
|
||||||
|
"pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};" \
|
||||||
|
"storage-api public capability contract re-export"
|
||||||
require_source_line \
|
require_source_line \
|
||||||
"crates/storage-api/src/lib.rs" \
|
"crates/storage-api/src/lib.rs" \
|
||||||
"pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};" \
|
"pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};" \
|
||||||
"storage-api public multipart DTO re-export"
|
"storage-api public multipart DTO re-export"
|
||||||
|
require_source_line \
|
||||||
|
"crates/storage-api/src/lib.rs" \
|
||||||
|
"pub use observability::{" \
|
||||||
|
"storage-api public observability contract re-export"
|
||||||
require_source_line \
|
require_source_line \
|
||||||
"crates/storage-api/src/lib.rs" \
|
"crates/storage-api/src/lib.rs" \
|
||||||
"pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};" \
|
"pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};" \
|
||||||
@@ -240,6 +252,10 @@ require_source_line \
|
|||||||
"crates/storage-api/src/lib.rs" \
|
"crates/storage-api/src/lib.rs" \
|
||||||
"pub use error::{StorageErrorCode, StorageResult};" \
|
"pub use error::{StorageErrorCode, StorageResult};" \
|
||||||
"storage-api public error contract re-export"
|
"storage-api public error contract re-export"
|
||||||
|
require_source_line \
|
||||||
|
"crates/storage-api/src/lib.rs" \
|
||||||
|
"pub use topology::{" \
|
||||||
|
"storage-api public topology contract re-export"
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
@@ -403,6 +419,66 @@ if [[ -s "$DIRECT_ECSTORE_IMPORT_HITS_FILE" ]]; then
|
|||||||
report_failure "direct rustfs_ecstore imports outside compatibility boundaries are forbidden: $(paste -sd '; ' "$DIRECT_ECSTORE_IMPORT_HITS_FILE")"
|
report_failure "direct rustfs_ecstore imports outside compatibility boundaries are forbidden: $(paste -sd '; ' "$DIRECT_ECSTORE_IMPORT_HITS_FILE")"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
(
|
||||||
|
cd "$ROOT_DIR"
|
||||||
|
rg -n --no-heading '\bstorage_compat::ecstore\b|\bmod\s+ecstore\b' \
|
||||||
|
crates/e2e_test/src crates/heal/tests crates/scanner/tests fuzz/fuzz_targets \
|
||||||
|
--glob '*.rs' \
|
||||||
|
--glob '!target/**' || true
|
||||||
|
) >"$TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE"
|
||||||
|
|
||||||
|
if [[ -s "$TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE" ]]; then
|
||||||
|
report_failure "test and fuzz storage compatibility harnesses must use direct aliases instead of nested ecstore modules: $(paste -sd '; ' "$TEST_HARNESS_NESTED_STORAGE_COMPAT_HITS_FILE")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
(
|
||||||
|
cd "$ROOT_DIR"
|
||||||
|
rg -n --no-heading '\bstorage_compat::ecstore\b|\bmod\s+ecstore\b' \
|
||||||
|
rustfs/src \
|
||||||
|
--glob '*.rs' \
|
||||||
|
--glob '!target/**' || true
|
||||||
|
) >"$RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE"
|
||||||
|
|
||||||
|
if [[ -s "$RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE" ]]; then
|
||||||
|
report_failure "RustFS runtime storage compatibility paths must use direct aliases instead of nested ecstore modules: $(paste -sd '; ' "$RUSTFS_NESTED_STORAGE_COMPAT_HITS_FILE")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
(
|
||||||
|
cd "$ROOT_DIR"
|
||||||
|
rg -n --no-heading '\bstorage_compat::(?:store|error|global|endpoints|set_disk|rpc|metrics_realtime|notification_sys|admin_server_info|store_utils|data_usage|disks_layout|event_notification)::|\bstorage_compat::disk::(?:RUSTFS_META_BUCKET|DiskAPI|endpoint::Endpoint)\b|\bstorage_compat::\{[^}\n]*(?:store|error|global|endpoints|set_disk|rpc|metrics_realtime|notification_sys|admin_server_info|store_utils|data_usage|disks_layout|event_notification|disk)::' \
|
||||||
|
rustfs/src \
|
||||||
|
--glob '*.rs' \
|
||||||
|
--glob '!target/**' || true
|
||||||
|
) >"$RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE"
|
||||||
|
|
||||||
|
if [[ -s "$RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE" ]]; then
|
||||||
|
report_failure "RustFS runtime scalar storage compatibility paths must use direct aliases instead of secondary modules: $(paste -sd '; ' "$RUSTFS_RUNTIME_SCALAR_STORAGE_COMPAT_HITS_FILE")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
(
|
||||||
|
cd "$ROOT_DIR"
|
||||||
|
{
|
||||||
|
rg -n --no-heading '\bstorage_compat::(?:bucket|config|rio|client|tier|compress|disk|rebalance)::' \
|
||||||
|
rustfs/src \
|
||||||
|
--glob '*.rs' \
|
||||||
|
--glob '!target/**' || true
|
||||||
|
rg -n --no-heading '\bpub\(crate\)\s+mod\s+(?:bucket|config|rio|client|tier|compress|disk|rebalance)\b' \
|
||||||
|
rustfs/src \
|
||||||
|
--glob '*storage_compat.rs' \
|
||||||
|
--glob '!target/**' || true
|
||||||
|
find rustfs/src -type f -name '*.rs' -print0 |
|
||||||
|
xargs -0 perl -0ne '
|
||||||
|
while (/use\s+crate::(?:app::|admin::|storage::)?storage_compat::\{[^;]*?\b(bucket|config|rio|client|tier|compress|disk|rebalance)::/sg) {
|
||||||
|
print "$ARGV:grouped storage_compat::$1 import\n";
|
||||||
|
}
|
||||||
|
'
|
||||||
|
}
|
||||||
|
) >"$RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE"
|
||||||
|
|
||||||
|
if [[ -s "$RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE" ]]; then
|
||||||
|
report_failure "RustFS runtime secondary storage compatibility paths must use direct aliases instead of bucket/config/rio/client/tier/compress/disk/rebalance modules: $(paste -sd '; ' "$RUSTFS_RUNTIME_SECONDARY_STORAGE_COMPAT_HITS_FILE")"
|
||||||
|
fi
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
rg -n --no-heading '#!\[allow\(unused_imports\)\]' \
|
rg -n --no-heading '#!\[allow\(unused_imports\)\]' \
|
||||||
|
|||||||
Reference in New Issue
Block a user