gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-16 09:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: consolidate storage owner boundary cleanup (#3921)

Browse Source 
* refactor: segment app usecase contracts by domain

* refactor: segment root storage contracts by domain

* refactor: segment root runtime facades by domain

* refactor: segment storage owner consumers by domain

* refactor: route storage owner runtime consumers by domain

* refactor: make storage owner rpc imports explicit

* refactor: remove storage owner wildcard imports (#3917)

* refactor: make storage owner root exports explicit (#3918)

* refactor: route storage facades through owner api (#3919)

* refactor: finalize storage owner phase branch
This commit is contained in:
Zhengchao An
2026-06-26 22:18:00 +08:00
committed by GitHub
parent 2c04807f05
commit 15c39db42f
68 changed files with 1717 additions and 630 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/storage_api.rs
+20 -20
View File
@@ -18,22 +18,22 @@ use std::sync::Arc;
use rustfs_storage_api as storage_contracts;
mod ecstore_bucket {
pub(crate) use crate::storage::ecstore_bucket::{
pub(crate) use crate::storage::storage_api::ecstore_bucket::{
bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning,
versioning_sys,
};
}
mod ecstore_capacity {
pub(crate) use crate::storage::ecstore_capacity::is_reserved_or_invalid_bucket;
pub(crate) use crate::storage::storage_api::ecstore_capacity::is_reserved_or_invalid_bucket;
}
mod ecstore_client {
pub(crate) use crate::storage::ecstore_client::admin_handler_utils;
pub(crate) use crate::storage::storage_api::ecstore_client::admin_handler_utils;
}
pub(crate) mod ecstore_cluster {
pub(crate) use crate::storage::ecstore_cluster::{
pub(crate) use crate::storage::storage_api::ecstore_cluster::{
ClusterDriveMembership, ClusterEndpointType, ClusterLocalNodeStorage, ClusterLocalNodeStorageSnapshot,
ClusterMembershipSnapshot, ClusterNodeMembership, ClusterPeerHealth, ClusterPeerHealthSnapshot, ClusterPoolState,
ClusterPoolStateSnapshot,
@@ -41,34 +41,34 @@ pub(crate) mod ecstore_cluster {
}
mod ecstore_config {
pub(crate) use crate::storage::ecstore_config::{com, init, storageclass};
pub(crate) use crate::storage::storage_api::ecstore_config::{com, init, storageclass};
}
#[allow(unused_imports)]
mod ecstore_disk {
pub(crate) use crate::storage::ecstore_disk::{RUSTFS_META_BUCKET, endpoint};
pub(crate) use crate::storage::storage_api::ecstore_disk::{RUSTFS_META_BUCKET, endpoint};
}
mod ecstore_error {
pub(crate) use crate::storage::ecstore_error::StorageError;
pub(crate) use crate::storage::storage_api::ecstore_error::StorageError;
}
#[allow(unused_imports)]
mod ecstore_layout {
pub(crate) use crate::storage::ecstore_layout::{EndpointServerPools, Endpoints, PoolEndpoints};
pub(crate) use crate::storage::storage_api::ecstore_layout::{EndpointServerPools, Endpoints, PoolEndpoints};
}
mod ecstore_metrics {
pub(crate) use crate::storage::ecstore_metrics::{CollectMetricsOpts, MetricType, collect_local_metrics};
pub(crate) use crate::storage::storage_api::ecstore_metrics::{CollectMetricsOpts, MetricType, collect_local_metrics};
}
mod ecstore_notification {
pub(crate) use crate::storage::ecstore_notification::NotificationSys;
pub(crate) use crate::storage::storage_api::ecstore_notification::NotificationSys;
}
#[allow(unused_imports)]
mod ecstore_rebalance {
pub(crate) use crate::storage::ecstore_rebalance::{
pub(crate) use crate::storage::storage_api::ecstore_rebalance::{
DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo,
RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, decode_rebalance_stop_propagation_record,
encode_rebalance_stop_propagation_record,
@@ -76,15 +76,15 @@ mod ecstore_rebalance {
}
mod ecstore_rpc {
pub(crate) use crate::storage::ecstore_rpc::PeerRestClient;
pub(crate) use crate::storage::storage_api::ecstore_rpc::PeerRestClient;
}
mod ecstore_storage {
pub(crate) use crate::storage::ecstore_storage::ECStore;
pub(crate) use crate::storage::storage_api::ecstore_storage::ECStore;
}
mod ecstore_tier {
pub(crate) use crate::storage::ecstore_tier::{tier, tier_admin, tier_config, tier_handlers};
pub(crate) use crate::storage::storage_api::ecstore_tier::{tier, tier_admin, tier_config, tier_handlers};
}
pub(crate) const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET;
@@ -414,15 +414,15 @@ pub(crate) mod data_usage {
use std::sync::Arc;
pub(crate) async fn load_data_usage_from_backend(
store: Arc<crate::storage::ECStore>,
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::StorageError> {
crate::storage::ecstore_data_usage::load_data_usage_from_backend(store).await
store: Arc<crate::storage::storage_api::ECStore>,
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
crate::storage::storage_api::ecstore_data_usage::load_data_usage_from_backend(store).await
}
}
pub(crate) mod access {
pub(crate) use crate::storage::access::{ReqInfo, authorize_request};
pub(crate) use crate::storage::request_context::{RequestContext, spawn_traced};
pub(crate) use crate::storage::storage_api::access_consumer::{ReqInfo, authorize_request};
pub(crate) use crate::storage::storage_api::request_context_consumer::{RequestContext, spawn_traced};
}
pub(crate) mod bucket {
@@ -501,7 +501,7 @@ pub(crate) mod metrics {
}
pub(crate) mod object {
pub(crate) use crate::storage::StorageObjectOptions;
pub(crate) use crate::storage::storage_api::StorageObjectOptions;
}
pub(crate) mod rebalance {
rustfs/src/app/admin_usecase.rs
+1 -1
View File
@@ -14,11 +14,11 @@
//! Admin application use-case contracts.
use super::storage_api::admin_usecase::StorageAdminApi;
use super::storage_api::admin_usecase::admin::get_server_info;
use super::storage_api::admin_usecase::capacity::{
PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free,
};
use super::storage_api::admin_usecase::contract::StorageAdminApi;
use super::storage_api::admin_usecase::data_usage::{
apply_bucket_usage_memory_overlay, load_data_usage_from_backend, refresh_versioned_bucket_usage_from_object_layer,
replace_bucket_usage_memory_from_info,
rustfs/src/app/bucket_usecase.rs
+6 -4
View File
@@ -37,6 +37,12 @@ use super::storage_api::bucket_usecase::bucket::{
utils::serialize,
versioning_sys::BucketVersioningSys,
};
use super::storage_api::bucket_usecase::contract::bucket::{
BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
};
use super::storage_api::bucket_usecase::contract::list::{
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _,
};
use super::storage_api::bucket_usecase::data_usage::remove_bucket_usage_from_backend;
use super::storage_api::bucket_usecase::error::StorageError;
use super::storage_api::bucket_usecase::helper::{OperationHelper, spawn_background_with_context};
@@ -46,10 +52,6 @@ use super::storage_api::bucket_usecase::s3_api::bucket::{
build_list_objects_output, build_list_objects_v2_output, parse_list_object_versions_params, parse_list_objects_v2_params,
rustfs_owner,
};
use super::storage_api::bucket_usecase::{
BucketOperations, BucketOptions, DeleteBucketOptions, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _, MakeBucketOptions,
};
use super::storage_api::bucket_usecase::{
get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
request_context, validate_list_object_unordered_with_delimiter,
rustfs/src/app/capacity_dirty_scope_test.rs
+3 -1
View File
@@ -13,7 +13,9 @@
// limitations under the License.
use super::storage_api::test::bucket::metadata_sys;
use super::storage_api::test::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _};
use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions, MakeBucketOptions};
use super::storage_api::test::contract::heal::HealOperations as _;
use super::storage_api::test::contract::object::ObjectIO as _;
use super::storage_api::test::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints};
use rustfs_common::heal_channel::{HealOpts, HealScanMode};
use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager};
rustfs/src/app/lifecycle_transition_api_test.rs
+6 -4
View File
@@ -18,13 +18,15 @@ use super::storage_api::test::bucket::{
metadata_sys,
transition_api::{ReadCloser, ReaderImpl},
};
use super::storage_api::test::contract::{
bucket::{BucketOperations, BucketOptions, MakeBucketOptions},
list::ListOperations as _,
multipart::MultipartOperations as _,
object::{ObjectIO as _, ObjectOperations as _},
};
use super::storage_api::test::ecfs::FS;
use super::storage_api::test::object_utils::to_s3s_etag;
use super::storage_api::test::runtime::{AppWarmBackend, TierConfig, TierType, WarmBackendGetOpts};
use super::storage_api::test::{
BucketOperations, BucketOptions, ListOperations as _, MakeBucketOptions, MultipartOperations as _, ObjectIO as _,
ObjectOperations as _,
};
use super::storage_api::test::{
ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, StorageObjectInfo as ObjectInfo,
StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader,
rustfs/src/app/multipart_usecase.rs
+5 -5
View File
@@ -15,8 +15,6 @@
//! Multipart application use-case contracts.
use super::storage_api::multipart_usecase::ECStore;
#[cfg(test)]
use super::storage_api::multipart_usecase::HTTPPreconditions;
use super::storage_api::multipart_usecase::access::has_bypass_governance_header;
use super::storage_api::multipart_usecase::bucket::quota::checker::QuotaChecker;
use super::storage_api::multipart_usecase::bucket::{
@@ -27,6 +25,11 @@ use super::storage_api::multipart_usecase::bucket::{
versioning_sys::BucketVersioningSys,
};
use super::storage_api::multipart_usecase::compression::is_disk_compressible;
#[cfg(test)]
use super::storage_api::multipart_usecase::contract::http::HTTPPreconditions;
use super::storage_api::multipart_usecase::contract::multipart::{CompletePart, MultipartOperations as _, MultipartUploadResult};
use super::storage_api::multipart_usecase::contract::object::{ObjectIO as _, ObjectOperations as _};
use super::storage_api::multipart_usecase::contract::range::HTTPRangeSpec;
use super::storage_api::multipart_usecase::data_usage::{
record_bucket_object_version_write_memory, record_bucket_object_write_memory,
};
@@ -51,9 +54,6 @@ use super::storage_api::multipart_usecase::sse::{
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in, map_get_object_reader_error,
mark_encrypted_multipart_metadata, sse_decryption, sse_prepare_encryption,
};
use super::storage_api::multipart_usecase::{
CompletePart, HTTPRangeSpec, MultipartOperations as _, MultipartUploadResult, ObjectIO as _, ObjectOperations as _,
};
use super::storage_api::multipart_usecase::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader};
use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write};
use crate::app::runtime_sources::{AppContext, current_app_context, resolve_object_store_handle_for_context};
rustfs/src/app/object_usecase.rs
+5 -3
View File
@@ -16,8 +16,6 @@
// Performance metrics recording (with zero-copy-metrics integration)
use super::storage_api::object_usecase::ECStore;
#[cfg(test)]
use super::storage_api::object_usecase::HTTPPreconditions;
use super::storage_api::object_usecase::access::{
PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut,
};
@@ -50,6 +48,11 @@ use super::storage_api::object_usecase::concurrency::{
self, ConcurrencyManager, GetObjectGuard, PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager,
get_put_concurrency_aware_buffer_size,
};
#[cfg(test)]
use super::storage_api::object_usecase::contract::http::HTTPPreconditions;
use super::storage_api::object_usecase::contract::namespace::NamespaceLocking;
use super::storage_api::object_usecase::contract::object::{ObjectIO as _, ObjectOperations as _};
use super::storage_api::object_usecase::contract::range::HTTPRangeSpec;
use super::storage_api::object_usecase::data_usage::{
record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory,
record_bucket_object_write_memory,
@@ -78,7 +81,6 @@ use super::storage_api::object_usecase::sse::{
};
use super::storage_api::object_usecase::storage_class as storageclass;
use super::storage_api::object_usecase::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper};
use super::storage_api::object_usecase::{HTTPRangeSpec, NamespaceLocking, ObjectIO as _, ObjectOperations as _};
use super::storage_api::object_usecase::{
RFC1123, check_preconditions, get_validated_store, has_replication_rules, parse_object_lock_legal_hold,
parse_object_lock_retention, parse_part_number_i32_to_usize, remove_object_lock_metadata_for_copy,
rustfs/src/app/select_object.rs
+1 -1
View File
@@ -1,4 +1,4 @@
use super::storage_api::select_object::ObjectOperations as _;
use super::storage_api::select_object::contract::object::ObjectOperations as _;
use super::storage_api::select_object::options::get_opts;
use super::storage_api::select_object::request_context::spawn_traced;
use super::storage_api::select_object::{get_validated_store, validate_sse_headers_for_read, validate_ssec_for_read};
rustfs/src/app/storage_api.rs
+350 -230
View File
File diff suppressed because it is too large Load Diff
rustfs/src/capacity/service.rs
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_api::capacity::{all_local_disk, disk_drive_path, disk_endpoint};
use crate::storage_api::capacity::service::{all_local_disk, disk_drive_path, disk_endpoint};
use rustfs_io_metrics::capacity_metrics::{
record_capacity_cache_hit, record_capacity_cache_miss, record_capacity_cache_served, record_capacity_refresh_request,
record_capacity_scan_mode,
rustfs/src/cluster_snapshot.rs
+6 -4
View File
@@ -16,11 +16,13 @@ use crate::runtime_capabilities::runtime_observability_snapshot;
use crate::server::{
DependencyReadiness, DependencyReadinessReport, ReadinessDegradedReason, snapshot_dependency_readiness_report,
};
use crate::storage_api::cluster::EndpointServerPools;
use crate::storage_api::cluster::contract::observability::ObservabilitySnapshot;
use crate::storage_api::cluster::contract::topology::TopologySnapshot;
use crate::storage_api::cluster::control_plane::{
ClusterControlPlane, ClusterControlPlaneSnapshot, ClusterLocalNodeStorageSnapshot, ClusterMembershipSnapshot,
ClusterPeerHealthSnapshot, ClusterPoolStateSnapshot,
};
use crate::storage_api::cluster::{EndpointServerPools, ObservabilitySnapshot, TopologySnapshot};
use crate::workload_admission::workload_admission_registry_snapshot;
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot};
@@ -115,9 +117,9 @@ pub fn cluster_has_actionable_pressure(snapshot: &ClusterReadOnlySnapshot) -> bo
#[cfg(test)]
mod tests {
use super::*;
use crate::storage_api::cluster::{
CapabilityState, CapabilityStatus, DiskCapabilities, Endpoint, Endpoints, PoolEndpoints, TopologyCapabilities,
};
use crate::storage_api::cluster::contract::capability::{CapabilityState, CapabilityStatus};
use crate::storage_api::cluster::contract::topology::{DiskCapabilities, TopologyCapabilities};
use crate::storage_api::cluster::{Endpoint, Endpoints, PoolEndpoints};
use rustfs_concurrency::{WorkloadAdmissionSnapshot, WorkloadClass};
#[test]
rustfs/src/error.rs
+2 -1
View File
@@ -12,7 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_api::error::{HTTPRangeError, QuotaError, StorageError};
use crate::storage_api::error::contract::range::HTTPRangeError;
use crate::storage_api::error::{QuotaError, StorageError};
use s3s::{S3Error, S3ErrorCode};
#[derive(Debug)]
rustfs/src/init.rs
+6 -6
View File
@@ -14,7 +14,7 @@
use crate::runtime_sources::{resolve_notify_interface, resolve_region};
use crate::server::ShutdownHandle;
use crate::storage_api::startup::{
use crate::storage_api::startup::init::{
get_bucket_notification_config, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
};
use crate::{admin, config, startup_runtime_sources, version};
@@ -717,7 +717,7 @@ where
/// When enabled, it spawns a background task that tunes concurrency settings
/// every 60 seconds.
pub async fn init_auto_tuner(ctx: tokio_util::sync::CancellationToken) {
use crate::storage_api::startup::concurrency::get_concurrency_manager;
use crate::storage_api::startup::init::concurrency::get_concurrency_manager;
use rustfs_io_metrics::AutoTuner;
use rustfs_io_metrics::TunerConfig;
use tracing::{debug, error, info};
@@ -850,7 +850,7 @@ pub async fn init_ftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::er
config.validate().await?;
// Create FTP server with protocol storage client
let fs = crate::storage_api::startup::ecfs::FS::new();
let fs = crate::storage_api::startup::init::ecfs::FS::new();
let storage_client = ProtocolStorageClient::new(fs);
let server: FtpsServer<ProtocolStorageClient> = FtpsServer::new(config, storage_client).await?;
let bind_addr = server.config().bind_addr;
@@ -970,7 +970,7 @@ pub async fn init_ftps_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
config.validate().await?;
// Create FTPS server with protocol storage client
let fs = crate::storage_api::startup::ecfs::FS::new();
let fs = crate::storage_api::startup::init::ecfs::FS::new();
let storage_client = ProtocolStorageClient::new(fs);
let server: FtpsServer<ProtocolStorageClient> = FtpsServer::new(config, storage_client).await?;
let bind_addr = server.config().bind_addr;
@@ -1088,7 +1088,7 @@ pub async fn init_webdav_system() -> Result<Option<ShutdownHandle>, Box<dyn std:
};
// Create WebDAV server with protocol storage client
let fs = crate::storage_api::startup::ecfs::FS::new();
let fs = crate::storage_api::startup::init::ecfs::FS::new();
let storage_client = ProtocolStorageClient::new(fs);
let server: WebDavServer<crate::protocols::ProtocolStorageClient> = WebDavServer::new(config, storage_client).await?;
let bind_addr = server.config().bind_addr;
@@ -1222,7 +1222,7 @@ pub async fn init_sftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
// file has insecure permissions.
let host_keys = SftpConfig::load_host_keys(&config.host_key_dir).await?;
let fs = crate::storage_api::startup::ecfs::FS::new();
let fs = crate::storage_api::startup::init::ecfs::FS::new();
let storage_client = ProtocolStorageClient::new(fs);
let server = SftpServer::new(config.clone(), storage_client, host_keys)?;
rustfs/src/protocols/client.rs
+3 -3
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::runtime_sources::resolve_action_credentials;
use crate::storage_api::protocols::ecfs::FS;
use crate::storage_api::protocols::client::{FS, ReqInfo, RequestContext};
use http::{HeaderMap, Method};
use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode};
use rustfs_credentials;
@@ -151,7 +151,7 @@ impl ProtocolStorageClient {
secret_key: params.secret_key.to_string().into(),
});
extensions.insert(crate::storage_api::protocols::access::ReqInfo {
extensions.insert(ReqInfo {
cred: Some(rustfs_credentials::Credentials {
access_key: params.access_key.to_string(),
secret_key: params.secret_key.to_string(),
@@ -169,7 +169,7 @@ impl ProtocolStorageClient {
object: params.object,
version_id: None,
region: None,
request_context: Some(crate::storage_api::protocols::request_context::RequestContext::fallback()),
request_context: Some(RequestContext::fallback()),
});
let req = S3Request {
rustfs/src/runtime_capabilities.rs
+9 -7
View File
@@ -12,11 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_api::cluster::{
CapabilitySnapshotError, CapabilityStatus, DiskCapabilities, EndpointServerPools, MemorySamplingState, ObservabilitySnapshot,
ObservabilitySnapshotProvider, PlatformSupport, TopologyCapabilities, TopologySnapshot, TopologySnapshotProvider,
UserspaceProfilingCapability, topology_snapshot_from_endpoint_pools_with_capabilities,
use crate::storage_api::cluster::contract::{
capability::{CapabilitySnapshotError, CapabilityStatus, MemorySamplingState, PlatformSupport, UserspaceProfilingCapability},
observability::{ObservabilitySnapshot, ObservabilitySnapshotProvider},
topology::{DiskCapabilities, TopologyCapabilities, TopologySnapshot, TopologySnapshotProvider},
};
use crate::storage_api::cluster::{EndpointServerPools, topology_snapshot_from_endpoint_pools_with_capabilities};
const NOT_WIRED_INTO_RUNTIME: &str = "not wired into runtime";
const EBPF_LINUX_ONLY: &str = "eBPF support is only available on linux targets";
@@ -184,9 +185,10 @@ fn cgroup_memory_status() -> CapabilityStatus {
#[cfg(test)]
mod tests {
use super::*;
use crate::storage_api::cluster::{
CapabilityState, Endpoint, Endpoints, ObservabilitySnapshotProvider, PoolEndpoints, TopologySnapshotProvider,
};
use crate::storage_api::cluster::contract::capability::CapabilityState;
use crate::storage_api::cluster::contract::observability::ObservabilitySnapshotProvider;
use crate::storage_api::cluster::contract::topology::TopologySnapshotProvider;
use crate::storage_api::cluster::{Endpoint, Endpoints, PoolEndpoints};
#[tokio::test]
async fn observability_provider_returns_platform_snapshot() {
rustfs/src/server/event.rs
+3 -2
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store, runtime_sources};
use crate::storage_api::server::{EventArgs as EcstoreEventArgs, StorageObjectInfo, register_event_dispatch_hook};
use crate::storage_api::server::event::{EventArgs as EcstoreEventArgs, StorageObjectInfo, register_event_dispatch_hook};
use chrono::{DateTime, Utc};
use rustfs_notify::{EventArgs as NotifyEventArgs, NotifyObjectInfo};
use rustfs_s3_types::EventName;
@@ -234,7 +234,8 @@ pub async fn init_event_notifier() {
#[cfg(test)]
mod tests {
use super::{convert_ecstore_object_info, parse_host_and_port};
use crate::storage_api::server::{StorageObjectInfo, TransitionedObject};
use crate::storage_api::server::event::StorageObjectInfo;
use crate::storage_api::server::event::contract::lifecycle::TransitionedObject;
use chrono::{DateTime, Utc};
use std::{collections::HashMap, sync::Arc};
use time::{Duration, OffsetDateTime};
rustfs/src/server/http.rs
+9 -7
View File
@@ -30,11 +30,11 @@ use crate::server::{
TlsAcceptorHolder, TlsHandshakeFailureKind, build_acceptor_from_loaded, load_tls_material, spawn_reload_loop,
},
};
use crate::storage_api::server as storage;
use crate::storage_api::server::request_context::{RequestContext, extract_request_id_from_headers};
use crate::storage_api::server::rpc::InternodeRpcService;
use crate::storage_api::server::tonic_service::make_server;
use crate::storage_api::server::{TONIC_RPC_PREFIX, verify_rpc_signature};
use crate::storage_api::server::http as storage;
use crate::storage_api::server::http::request_context::{RequestContext, extract_request_id_from_headers};
use crate::storage_api::server::http::rpc::InternodeRpcService;
use crate::storage_api::server::http::tonic_service::make_server;
use crate::storage_api::server::http::{TONIC_RPC_PREFIX, verify_rpc_signature};
use bytes::Bytes;
use http::{HeaderMap, Method, Request as HttpRequest, Response};
use hyper::body::Incoming;
@@ -1104,7 +1104,8 @@ fn process_connection(
.layer(
TraceLayer::new_for_http()
.make_span_with(|request: &HttpRequest<_>| {
let request_context = request.extensions().get::<crate::storage_api::server::request_context::RequestContext>();
let request_context =
request.extensions().get::<crate::storage_api::server::http::request_context::RequestContext>();
let request_id = request_context
.map(|ctx| ctx.request_id.as_str())
.unwrap_or("unknown");
@@ -1253,7 +1254,8 @@ fn process_connection(
.layer(
TraceLayer::new_for_http()
.make_span_with(|request: &HttpRequest<_>| {
let request_context = request.extensions().get::<crate::storage_api::server::request_context::RequestContext>();
let request_context =
request.extensions().get::<crate::storage_api::server::http::request_context::RequestContext>();
let request_id = request_context
.map(|ctx| ctx.request_id.as_str())
.unwrap_or("unknown");
rustfs/src/server/layer.rs
+2 -2
View File
@@ -25,8 +25,8 @@ use crate::server::{
MINIO_HEALTH_READY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX, active_http_requests, has_path_prefix, is_admin_path,
is_table_catalog_path,
};
use crate::storage_api::server::apply_cors_headers;
use crate::storage_api::server::request_context::{
use crate::storage_api::server::layer::apply_cors_headers;
use crate::storage_api::server::layer::request_context::{
RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced,
};
use bytes::Bytes;
rustfs/src/server/module_switch.rs
+1 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use super::runtime_sources;
use crate::storage_api::server::{Error as StorageError, read_config, save_config};
use crate::storage_api::server::module_switch::{Error as StorageError, read_config, save_config};
use serde::{Deserialize, Serialize};
use std::sync::atomic::{AtomicBool, Ordering};
rustfs/src/server/readiness.rs
+3 -2
View File
@@ -15,9 +15,10 @@
use crate::server::runtime_sources;
use crate::server::{ServiceState, ServiceStateManager};
use crate::server::{has_path_prefix, is_table_catalog_path};
use crate::storage_api::server::{Endpoint, EndpointServerPools, StorageAdminApi, is_dist_erasure};
use crate::storage_api::server::readiness::contract::admin::StorageAdminApi;
use crate::storage_api::server::readiness::{Endpoint, EndpointServerPools, is_dist_erasure};
#[cfg(test)]
use crate::storage_api::server::{Endpoints, PoolEndpoints};
use crate::storage_api::server::readiness::{Endpoints, PoolEndpoints};
use bytes::Bytes;
use http::{Request as HttpRequest, Response, StatusCode};
use http_body::Body;
rustfs/src/server/runtime_sources.rs
+1 -1
View File
@@ -16,7 +16,7 @@ use crate::runtime_sources::{
NotifyInterface, resolve_endpoints_handle, resolve_iam_ready, resolve_kms_runtime_service_manager,
resolve_lock_clients_handle, resolve_notify_interface, resolve_object_store_handle, resolve_server_config,
};
use crate::storage_api::server::{ECStore, EndpointServerPools};
use crate::storage_api::server::runtime_sources::{ECStore, EndpointServerPools};
use rustfs_kms::KmsServiceManager;
use rustfs_lock::LockClient;
use std::collections::HashMap;
rustfs/src/startup_background.rs
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_api::startup::{ECStore, set_workload_admission_snapshot_provider};
use crate::storage_api::startup::background::{ECStore, set_workload_admission_snapshot_provider};
use crate::workload_admission::RustFsWorkloadAdmissionSnapshotProvider;
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
use rustfs_heal::{
rustfs/src/startup_bucket_metadata.rs
+3 -2
View File
@@ -13,8 +13,9 @@
// limitations under the License.
use crate::startup_runtime_sources;
use crate::storage_api::startup::{
BucketOperations, BucketOptions, ECStore, init_bucket_metadata_sys, try_migrate_bucket_metadata, try_migrate_iam_config,
use crate::storage_api::startup::bucket_metadata::contract::bucket::{BucketOperations, BucketOptions};
use crate::storage_api::startup::bucket_metadata::{
ECStore, init_bucket_metadata_sys, try_migrate_bucket_metadata, try_migrate_iam_config,
};
use std::{
io::{Error, Result},
rustfs/src/startup_deadlock.rs
+1 -1
View File
@@ -19,7 +19,7 @@ const LOG_SUBSYSTEM_STARTUP: &str = "startup";
const EVENT_DEADLOCK_DETECTOR_STATE: &str = "deadlock_detector_state";
pub(crate) fn init_deadlock_detector_runtime() {
let detector = crate::storage_api::startup::deadlock_detector::get_deadlock_detector();
let detector = crate::storage_api::startup::deadlock::get_deadlock_detector();
if detector.is_enabled() {
detector.start();
info!(
rustfs/src/startup_fs_guard.rs
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_api::startup::EndpointServerPools;
use crate::storage_api::startup::fs_guard::EndpointServerPools;
use rustfs_config::{
DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, ENV_RUSTFS_UNSUPPORTED_FS_POLICY, RUSTFS_UNSUPPORTED_FS_POLICY_FAIL,
RUSTFS_UNSUPPORTED_FS_POLICY_WARN,
rustfs/src/startup_iam.rs
+1 -1
View File
@@ -14,7 +14,7 @@
use crate::runtime_sources::AppContext;
use crate::server::{ServiceStateManager, publish_ready_when_runtime_ready};
use crate::storage_api::startup::ECStore;
use crate::storage_api::startup::iam::ECStore;
use rustfs_common::{GlobalReadiness, SystemStage};
use rustfs_iam::init_iam_sys;
use rustfs_kms::KmsServiceManager;
rustfs/src/startup_lifecycle.rs
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_api::startup::ECStore;
use crate::storage_api::startup::lifecycle::ECStore;
use crate::{
server::{ServiceStateManager, ShutdownHandle, wait_for_shutdown},
startup_iam::{IamBootstrapDisposition, publish_ready_for_iam_bootstrap},
rustfs/src/startup_notification.rs
+2 -2
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::init::add_bucket_notification_configuration;
use crate::storage_api::startup::{EndpointServerPools, Result as StorageResult, new_global_notification_sys};
use crate::storage_api::startup::notification::{EndpointServerPools, Result as StorageResult, new_global_notification_sys};
use std::{
future::Future,
io::{Error, Result},
@@ -76,7 +76,7 @@ fn log_embedded_optional_service_skipped(service: &str, err: impl std::fmt::Disp
#[cfg(test)]
mod tests {
use super::init_notification_system_with;
use crate::storage_api::startup::Error as EcstoreError;
use crate::storage_api::startup::notification::Error as EcstoreError;
#[tokio::test]
async fn notification_system_returns_source_error() {
rustfs/src/startup_runtime_sources.rs
+1 -1
View File
@@ -14,7 +14,7 @@
use crate::config::RustFSBufferConfig;
use crate::runtime_sources::{resolve_outbound_tls_generation, resolve_replication_pool_handle};
use crate::storage_api::startup::{DynReplicationPool, set_global_region, set_global_rustfs_port};
use crate::storage_api::startup::runtime_sources::{DynReplicationPool, set_global_region, set_global_rustfs_port};
use rustfs_kms::KmsServiceManager;
use rustfs_obs::{GlobalError as ObservabilityError, OtelGuard};
use rustfs_tls_runtime::{OutboundTlsMaterial, TlsGeneration};
rustfs/src/startup_services.rs
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_api::startup::{ECStore, EndpointServerPools};
use crate::storage_api::startup::services::{ECStore, EndpointServerPools};
use crate::{
config::Config,
init::{init_buffer_profile_system, init_kms_system},
rustfs/src/startup_shutdown.rs
+1 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage_api::startup::shutdown_background_services;
use crate::storage_api::startup::shutdown::shutdown_background_services;
use crate::{
server::{ServiceState, ServiceStateManager, ShutdownHandle, ShutdownSignal, shutdown_event_notifier, stop_audit_system},
startup_optional_runtime_sidecars::{
rustfs/src/startup_storage.rs
+1 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
use crate::storage_api::startup::{
use crate::storage_api::startup::storage::{
ECStore, EndpointServerPools, init_background_replication, init_ecstore_config, init_global_config_sys, init_local_disks,
init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints, try_migrate_server_config, update_erasure_type,
};
rustfs/src/storage/access.rs
+16 -3
View File
@@ -22,9 +22,9 @@ use crate::auth::{check_key_valid, get_condition_values_with_query_and_client_in
use crate::error::ApiError;
use crate::license::license_check;
use crate::server::RemoteAddr;
use crate::storage::contract::bucket::BucketOperations;
use crate::storage::request_context::RequestContext;
use crate::storage::runtime_sources;
use crate::storage::storage_api::access_consumer::contract::bucket::BucketOperations;
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use metrics::counter;
use rustfs_iam::error::Error as IamError;
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
@@ -2051,10 +2051,23 @@ impl S3Access for FS {
}
#[cfg(test)]
#[allow(unused_imports)]
mod tests {
use super::*;
use super::{
AMZ_WRITE_OFFSET_BYTES_HEADER, BucketPolicyArgs, BucketPolicyExistingObjectTagHint, BucketPolicyRawLoadErrorKind, FS,
ObjectTagConditions, PostObjectRequestMarker, ReqInfo, S3Access, StorageError,
bucket_policy_needs_existing_object_tag_from_hint, classify_bucket_policy_raw_load_error,
complete_multipart_upload_authorize_action, get_bucket_policy_authorize_action, has_write_offset_bytes_header,
legal_hold_write_requested, list_parts_authorize_action, load_bucket_policy_existing_object_tag_hint,
merge_list_bucket_query_conditions, owner_can_bypass_policy_deny, post_object_authorize_action,
put_bucket_policy_authorize_action, request_context_from_req, retention_write_requested, secondary_tag_hint_action,
table_data_plane_admin_action, validate_post_object_success_controls,
};
use crate::error::ApiError;
use http::{Extensions, HeaderMap, Method, Uri};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_policy::policy::{BucketPolicy, bucket_policy_uses_existing_object_tag_conditions};
use s3s::{S3ErrorCode, S3Request, dto::*};
use std::collections::HashMap;
use time::OffsetDateTime;
rustfs/src/storage/backpressure.rs
+2 -1
View File
@@ -483,8 +483,9 @@ impl Default for BackpressureMonitor {
}
#[cfg(test)]
#[allow(unused_imports)]
mod tests {
use super::*;
use super::{BackpressureMonitor, BackpressurePipe, BackpressureState, ObjectPipeBackpressurePolicy};
#[test]
fn test_backpressure_config_default() {
rustfs/src/storage/concurrency/io_schedule.rs
+9 -1
View File
@@ -1930,9 +1930,17 @@ pub fn get_buffer_size_opt_in(file_size: i64) -> usize {
// ============================================
#[cfg(test)]
#[allow(unused_imports)]
mod tests {
use super::*;
use super::{
IoLoadLevel, IoPriority, IoPriorityMetrics, IoPriorityQueue, IoPriorityQueueConfig, IoSchedulerConfig,
IoSchedulingContext, IoStrategy, get_advanced_buffer_size, get_buffer_size_opt_in, get_concurrency_aware_buffer_size,
};
use rustfs_io_core::io_profile::{AccessPattern, StorageMedia};
use rustfs_io_metrics::bandwidth::{BandwidthSnapshot, BandwidthTier};
use serial_test::serial;
use std::sync::atomic::Ordering;
use std::time::Duration;
#[tokio::test]
#[serial]
rustfs/src/storage/concurrency/manager.rs
+9 -2
View File
@@ -19,7 +19,7 @@ use super::io_schedule::{
get_advanced_buffer_size,
};
use super::request_guard::{GetObjectGuard, PutObjectGuard};
use crate::storage::runtime_sources;
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use rustfs_concurrency::{
AdmissionState, GetObjectQueueSnapshot, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot,
WorkloadAdmissionSnapshotProvider, WorkloadClass,
@@ -664,9 +664,16 @@ impl Default for ConcurrencyManager {
// ============================================
#[cfg(test)]
#[allow(unused_imports)]
mod integration_tests {
use super::*;
use super::super::io_schedule::{IoLoadLevel, IoPriority};
use super::super::request_guard::GetObjectGuard;
use super::ConcurrencyManager;
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshotProvider, WorkloadClass};
use rustfs_config::MI_B;
use rustfs_io_core::io_profile::{AccessPattern, StorageMedia};
use serial_test::serial;
use std::time::Duration;
#[tokio::test]
#[serial]
rustfs/src/storage/concurrency/request_guard.rs
+1 -1
View File
@@ -173,7 +173,7 @@ impl Drop for PutObjectGuard {
#[cfg(test)]
mod tests {
use super::*;
use super::{GetObjectGuard, PutObjectGuard};
#[test]
fn test_guard_increments_counter() {
rustfs/src/storage/deadlock_detector.rs
+3 -1
View File
@@ -611,8 +611,10 @@ pub fn is_deadlock_detection_enabled() -> bool {
}
#[cfg(test)]
#[allow(unused_imports)]
mod tests {
use super::*;
use super::{DeadlockDetector, LockInfo, LockType, RequestHangDetectionPolicy, RequestResourceTracker, WaitGraphEdge};
use std::time::{Duration, Instant};
#[test]
fn test_deadlock_detector_config_default() {
rustfs/src/storage/ecfs.rs
+9 -7
View File
@@ -23,15 +23,17 @@ use super::{
use super::{StorageReplicationConfigExt as _, StorageVersioningConfigExt as _};
use crate::error::ApiError;
use crate::storage::access::has_bypass_governance_header;
use crate::storage::contract::{
use crate::storage::helper::OperationHelper;
use crate::storage::options::get_opts;
use crate::storage::s3_api::{self, acl};
use crate::storage::storage_api::ecfs_consumer::contract::{
bucket::{BucketOperations, BucketOptions},
object::{ObjectLockRetentionOptions, ObjectOperations as _},
};
use crate::storage::helper::OperationHelper;
use crate::storage::options::get_opts;
use crate::storage::runtime_sources;
use crate::storage::s3_api::{self, acl};
use crate::storage::{parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled};
use crate::storage::storage_api::ecfs_consumer::object_lock::{
parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled,
};
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use crate::table_catalog;
use http::StatusCode;
use metrics::{counter, histogram};
@@ -52,7 +54,7 @@ const LOG_SUBSYSTEM_OBJECT: &str = "object";
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
const LOG_SUBSYSTEM_TAGGING: &str = "tagging";
use crate::storage::StorageObjectOptions as ObjectOptions;
use crate::storage::storage_api::ecfs_consumer::StorageObjectOptions as ObjectOptions;
#[derive(Debug, Clone)]
pub struct FS {
rustfs/src/storage/ecfs_extend.rs
+4 -4
View File
@@ -20,11 +20,11 @@ use super::{
use crate::config::{RustFSBufferConfig, WorkloadProfile, is_buffer_profile_enabled};
use crate::error::ApiError;
use crate::server::cors;
use crate::storage::contract::{
use crate::storage::ecfs::ListObjectUnorderedQuery;
use crate::storage::storage_api::ecfs_extend_consumer::contract::{
bucket::{BucketOperations, BucketOptions},
object::ObjectToDelete,
};
use crate::storage::ecfs::ListObjectUnorderedQuery;
use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE};
use http::{HeaderMap, HeaderValue, StatusCode};
use metrics::counter;
@@ -49,8 +49,8 @@ use time::format_description::well_known::Rfc3339;
use time::{format_description::FormatItem, macros::format_description};
use tracing::{debug, warn};
use crate::storage::StorageObjectInfo as ObjectInfo;
use crate::storage::runtime_sources;
use crate::storage::storage_api::ecfs_extend_consumer::StorageObjectInfo as ObjectInfo;
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
const LOG_COMPONENT_STORAGE: &str = "storage";
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
rustfs/src/storage/ecfs_test.rs
+7 -9
View File
@@ -14,16 +14,17 @@
#[cfg(test)]
mod tests {
use super::super::{BucketMetadata, DEFAULT_READ_BUFFER_SIZE, get_global_bucket_metadata_sys, set_bucket_metadata};
use crate::config::WorkloadProfile;
use crate::server::cors;
use crate::storage::ecfs::{FS, validate_object_lock_configuration_input};
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
use crate::storage::{
StorageObjectInfo as ObjectInfo, apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions,
get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal, matches_origin_pattern, parse_etag,
parse_object_lock_legal_hold, parse_object_lock_retention, process_lambda_configurations, process_queue_configurations,
process_topic_configurations, remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata,
use crate::storage::storage_api::test_consumer::{
BucketMetadata, DEFAULT_READ_BUFFER_SIZE, StorageObjectInfo as ObjectInfo, apply_cors_headers,
apply_default_lock_retention_metadata, bucket_metadata_sys_initialized, check_preconditions, decode_tags_to_map,
get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, get_global_bucket_metadata_sys, is_etag_equal,
matches_origin_pattern, parse_etag, parse_object_lock_legal_hold, parse_object_lock_retention,
process_lambda_configurations, process_queue_configurations, process_topic_configurations,
remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata, set_bucket_metadata,
validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter,
};
use http::{Extensions, HeaderMap, HeaderValue, Method, StatusCode, Uri};
@@ -953,8 +954,6 @@ mod tests {
#[tokio::test]
async fn test_validate_bucket_object_lock_enabled() {
use super::super::bucket_metadata_sys_initialized;
use super::super::set_bucket_metadata;
use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
use time::OffsetDateTime;
@@ -1793,7 +1792,6 @@ mod tests {
/// with a single-element vec value, matching the format expected by policy evaluation.
#[test]
fn test_object_tag_condition_key_format() {
use super::super::decode_tags_to_map;
use std::collections::HashMap;
let tags_str = "security=public&project=webapp&env=prod";
rustfs/src/storage/head_prefix.rs
+1 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use super::ECStore;
use crate::storage::contract::list::ListOperations as _;
use crate::storage::storage_api::head_prefix_consumer::contract::list::ListOperations as _;
use std::sync::Arc;
/// Determines if the key "looks like a prefix" (ends with `/`).
rustfs/src/storage/helper.rs
+9 -3
View File
@@ -15,7 +15,7 @@
use crate::server::{convert_ecstore_object_info, is_audit_module_enabled, is_notify_module_enabled};
use crate::storage::access::{ReqInfo, request_context_from_req};
use crate::storage::request_context::{RequestContext, extract_request_id_from_headers};
use crate::storage::runtime_sources;
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use hashbrown::HashMap;
use http::StatusCode;
use metrics::counter;
@@ -37,7 +37,7 @@ use std::future::Future;
use tokio::runtime::{Builder, Handle};
use tracing::{Instrument, info_span, warn};
use crate::storage::StorageObjectInfo as ObjectInfo;
use crate::storage::storage_api::helper_consumer::StorageObjectInfo as ObjectInfo;
/// Schedules an asynchronous task on the current runtime;
/// if there is no runtime, creates a minimal runtime execution on a new thread.
@@ -391,12 +391,18 @@ impl Drop for OperationHelper {
}
#[cfg(test)]
#[allow(unused_imports)]
mod tests {
use super::*;
use super::OperationHelper;
use crate::server::{refresh_audit_module_enabled, refresh_notify_module_enabled};
use crate::storage::access::ReqInfo;
use crate::storage::request_context::RequestContext;
use http::{Extensions, HeaderMap, HeaderValue, Method, Uri};
use metrics::{Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Metadata, SharedString, Unit};
use rustfs_credentials::Credentials;
use rustfs_s3_ops::S3Operation;
use rustfs_s3_types::EventName;
use s3s::S3Request;
use s3s::dto::DeleteObjectTaggingInput;
use std::sync::{Arc, Mutex};
use temp_env::with_vars;
rustfs/src/storage/lock_optimizer.rs
+4 -1
View File
@@ -381,9 +381,12 @@ pub fn is_lock_optimization_enabled() -> bool {
}
#[cfg(test)]
#[allow(unused_imports)]
mod tests {
use super::*;
use super::{LockOptimizeConfig, LockOptimizer, LockStats, OptimizedLockGuard};
use std::sync::Mutex;
use std::sync::atomic::Ordering;
use std::time::Duration;
#[test]
fn test_lock_optimize_config_default() {
rustfs/src/storage/mod.rs
+45 -6
View File
@@ -40,12 +40,51 @@ mod multi_factor_scheduler_integration_test;
pub(crate) mod runtime_sources;
#[cfg(test)]
mod sse_test;
mod storage_api;
pub(crate) mod storage_api;
pub(crate) use ecfs_extend::*;
pub(crate) use sse::{
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, extract_server_side_encryption_from_headers, sse_decryption,
sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read,
validate_sse_headers_for_write, validate_ssec_for_read,
#[allow(unused_imports)]
pub(crate) use storage_api::{
BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_TABLE_CATALOG_META_PREFIX,
BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG, BUCKET_TABLE_RESERVED_PREFIX, BUCKET_VERSIONING_CONFIG,
BUCKET_WEBSITE_CONFIG, BucketBandwidthMonitor, BucketMetadata, BucketVersioningSys, CheckPartsResp, CollectMetricsOpts,
DEFAULT_READ_BUFFER_SIZE, DailyAllTierStats, DeleteOptions, DiskError, DiskInfo, DiskInfoOptions, DiskResult, DiskStore,
DynReader, DynReplicationPool, ECStore, Endpoint, EndpointServerPools, Error, EventArgs, ExpiryState, FS, FileInfoVersions,
FileReader, FileWriter, GetObjectReader, HashReader, LocalPeerS3Client, MetricType, NotificationSys, OBJECT_LOCK_CONFIG,
ObjectInfo, ObjectLockBlockReason, ObjectOptions, ObjectPartInfo, ObjectStoreResolver, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
PolicySys, PoolEndpoints, PutObjReader, QuotaError, RUSTFS_META_BUCKET, RawFileInfo, ReadMultipleReq, ReadMultipleResp,
ReadOptions, RenameDataResp, ReplicationStats, Result, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
SetupType, StorageDeletedObject, StorageDiskRpcExt, StorageError, StorageGetObjectReader, StorageObjectInfo,
StorageObjectOptions, StorageObjectToDelete, StoragePeerS3ClientExt, StoragePutObjReader, StorageReplicationConfigExt,
StorageVersioningConfigExt, TONIC_RPC_PREFIX, TierConfigMgr, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
WorkloadAdmissionSnapshotProviderRef, WriteEncryption, WritePlan, access_consumer, add_object_lock_years, all_local_disk,
all_local_disk_path, bucket_metadata_runtime_initialized, check_retention_for_modification, collect_local_metrics,
compression_metadata_value, contract, decode_tags, decode_tags_to_map, delete_bucket_metadata_config, disk_drive_path,
disk_endpoint, ecfs_consumer, ecfs_extend_consumer, ecstore_admin, ecstore_bucket, ecstore_capacity, ecstore_client,
ecstore_cluster, ecstore_compression, ecstore_config, ecstore_data_usage, ecstore_disk, ecstore_error, ecstore_event,
ecstore_global, ecstore_layout, ecstore_metrics, ecstore_notification, ecstore_rebalance, ecstore_rio, ecstore_rpc,
ecstore_set_disk, ecstore_storage, ecstore_tier, encode_tags, find_local_disk_by_ref, get_bucket_accelerate_config,
get_bucket_cors_config, get_bucket_logging_config, get_bucket_metadata, get_bucket_notification_config,
get_bucket_object_lock_config, get_bucket_policy_raw, get_bucket_replication_config, get_bucket_request_payment_config,
get_bucket_sse_config, get_bucket_website_config, get_daily_all_tier_stats, get_global_boot_time, get_global_bucket_monitor,
get_global_deployment_id, get_global_endpoints_opt, get_global_expiry_state, get_global_lock_client, get_global_lock_clients,
get_global_notification_sys, get_global_region, get_global_replication_pool, get_global_replication_stats,
get_global_tier_config_mgr, get_local_server_property, get_lock_acquire_timeout, get_public_access_block_config,
global_rustfs_port, head_prefix_consumer, helper_consumer, init_background_replication, init_bucket_metadata_sys,
init_ecstore_config, init_global_config_sys, init_local_disks, init_lock_clients, is_all_buckets_not_found, is_dist_erasure,
is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, is_valid_storage_class, load_bucket_metadata,
new_global_notification_sys, new_object_layer_fn, options_consumer, prewarm_local_disk_id_map, read_config,
record_replication_proxy, register_event_dispatch_hook, reload_transition_tier_config, replication_queue_current_count,
resolve_object_store_handle, rpc_consumer, runtime_sources_consumer, s3_api_consumer, save_config, serialize,
set_bucket_metadata, set_global_endpoints, set_global_region, set_global_rustfs_port, set_object_store_resolver,
set_workload_admission_snapshot_provider, shutdown_background_services, table_catalog_path_hash, to_s3s_etag,
topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, try_migrate_iam_config,
try_migrate_server_config, update_bucket_metadata_config, update_erasure_type, verify_rpc_signature, wrap_reader,
};
#[cfg(test)]
#[allow(unused_imports)]
pub(crate) use storage_api::{
BucketMetadataSys, DecryptReader, DisksLayout, EncryptReader, Endpoints, HardLimitReader, STORAGE_CLASS_SUB_SYS,
boxed_reader, get_global_bucket_metadata_sys, test_consumer,
};
pub(crate) use storage_api::*;
rustfs/src/storage/options.rs
+15 -6
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use super::{BucketVersioningSys, Result, StorageError};
use crate::storage::contract::{object::HTTPPreconditions, range::HTTPRangeSpec};
use crate::storage::storage_api::options_consumer::contract::{object::HTTPPreconditions, range::HTTPRangeSpec};
use http::header::{IF_MATCH, IF_NONE_MATCH};
use http::{HeaderMap, HeaderValue};
use rustfs_utils::http::{
@@ -44,10 +44,7 @@ use crate::auth::AuthType;
use crate::auth::get_query_param;
use crate::auth::get_request_auth_type_with_query;
use crate::auth::is_request_presigned_signature_v4_with_query;
use crate::storage::StorageObjectOptions as ObjectOptions;
#[cfg(test)]
use rustfs_utils::http::insert_header;
use crate::storage::storage_api::options_consumer::StorageObjectOptions as ObjectOptions;
/// Creates options for deleting an object in a bucket.
pub async fn del_opts(
@@ -782,12 +779,24 @@ fn get_content_sha256_cksum(headers: &HeaderMap<HeaderValue>, service_type: Serv
}
#[cfg(test)]
#[allow(unused_imports)]
mod tests {
use proptest::prelude::*;
use temp_env;
use super::*;
use super::super::StorageError;
use super::{
ENV_REJECT_ARCHIVE_CONTENT_ENCODING, SUPPORTED_HEADERS, copy_dst_opts, copy_src_opts, del_opts,
detect_content_type_from_object_name, extract_metadata, extract_metadata_from_mime,
extract_metadata_from_mime_with_object_name, filter_object_metadata, get_default_opts, get_opts, parse_copy_source_range,
put_opts, put_opts_from_headers, validate_archive_content_encoding,
};
use http::{HeaderMap, HeaderValue};
use rustfs_utils::http::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_REQUEST, insert_header,
};
use s3s::S3ErrorCode;
use std::collections::HashMap;
use uuid::Uuid;
rustfs/src/storage/request_context.rs
+4 -1
View File
@@ -192,8 +192,11 @@ where
}
#[cfg(test)]
#[allow(unused_imports)]
mod tests {
use super::*;
use super::{RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers};
use http::HeaderMap;
use opentelemetry::global;
use opentelemetry::trace::{SpanContext, TraceContextExt, TraceFlags, TraceId, TraceState, TracerProvider as _};
use opentelemetry_sdk::propagation::TraceContextPropagator;
use opentelemetry_sdk::trace::SdkTracerProvider;
rustfs/src/storage/rpc/bucket.rs
+11 -1
View File
@@ -12,7 +12,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use super::NodeService;
use crate::storage::storage_api::rpc_consumer::node_service::contract::bucket::{
BucketOptions, DeleteBucketOptions, MakeBucketOptions,
};
use crate::storage::storage_api::rpc_consumer::node_service::{
DiskError, StoragePeerS3ClientExt as _, load_bucket_metadata, set_bucket_metadata,
};
use rustfs_common::heal_channel::HealOpts;
use rustfs_protos::proto_gen::node_service::*;
use tonic::{Request, Response, Status};
use tracing::debug;
impl NodeService {
pub(super) async fn handle_delete_bucket_metadata(
rustfs/src/storage/rpc/disk.rs
+12 -3
View File
@@ -12,13 +12,22 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use crate::storage::runtime_sources;
use super::NodeService;
use crate::storage::storage_api::rpc_consumer::node_service::{
DeleteOptions, DiskError, DiskInfoOptions, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions,
StorageDiskRpcExt as _, UpdateMetadataOpts,
};
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use bytes::Bytes;
use rustfs_filemeta::FileInfo;
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC,
};
use rustfs_protos::proto_gen::node_service::*;
use serde::de::DeserializeOwned;
use std::io::Cursor;
use tonic::{Request, Response, Status};
use tracing::debug;
fn decode_msgpack_or_json<T: DeserializeOwned>(binary: &[u8], json: &str, value_name: &str) -> std::result::Result<T, DiskError> {
if !binary.is_empty() {
@@ -1036,7 +1045,7 @@ impl NodeService {
#[cfg(test)]
mod tests {
use super::*;
use super::{decode_msgpack_or_json, encode_msgpack};
use serde::{Deserialize, Serialize};
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
rustfs/src/storage/rpc/event.rs
+4 -1
View File
@@ -12,8 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use super::NodeService;
use bytes::Bytes;
use rustfs_notify::notification_system;
use rustfs_protos::proto_gen::node_service::*;
use tonic::{Request, Response, Status};
impl NodeService {
pub(super) async fn handle_get_live_events(
rustfs/src/storage/rpc/health.rs
+15 -3
View File
@@ -12,9 +12,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use super::NodeService;
use crate::storage::rpc::encode_msgpack_map;
use crate::storage::runtime_sources;
use crate::storage::storage_api::rpc_consumer::node_service::contract::admin::StorageAdminApi;
use crate::storage::storage_api::rpc_consumer::node_service::get_local_server_property;
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use bytes::Bytes;
use rustfs_madmin::health::{
get_cpus, get_mem_info, get_os_info, get_partitions, get_proc_info, get_sys_config, get_sys_errors, get_sys_services,
};
use rustfs_madmin::net::get_net_info;
use rustfs_protos::proto_gen::node_service::*;
use tonic::{Request, Response, Status};
impl NodeService {
pub(super) async fn handle_get_proc_info(
@@ -240,7 +249,10 @@ impl NodeService {
#[cfg(test)]
mod tests {
use super::*;
use crate::storage::rpc::encode_msgpack_map;
use rmp_serde::Deserializer;
use serde::Deserialize;
use std::io::Cursor;
#[test]
fn local_storage_info_rpc_payload_uses_msgpack_map_encoding() {
rustfs/src/storage/rpc/http_service.rs
+15 -7
View File
@@ -12,14 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::super::DEFAULT_READ_BUFFER_SIZE;
use super::super::StorageDiskRpcExt as _;
use super::super::WalkDirOptions;
use super::super::find_local_disk_by_ref;
use super::super::verify_rpc_signature;
use crate::server::RPC_PREFIX;
use crate::storage::request_context::spawn_traced;
use crate::storage::runtime_sources;
use crate::storage::storage_api::rpc_consumer::http_service::{
DEFAULT_READ_BUFFER_SIZE, StorageDiskRpcExt as _, WalkDirOptions, find_local_disk_by_ref, verify_rpc_signature,
};
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use bytes::{Bytes, BytesMut};
use futures_util::TryStreamExt;
use http::{HeaderMap, Method, Request, Response, StatusCode, Uri};
@@ -716,7 +714,17 @@ fn put_file_stage_error_message(stage: &str, query: &PutFileQuery, err: &dyn std
#[cfg(test)]
mod tests {
use super::*;
use super::{
LOG_SUBSYSTEM_DIRECTORY_WALK, LOG_SUBSYSTEM_FILE_TRANSFER, LOG_SUBSYSTEM_ROUTING, PUT_FILE_STREAM_PATH, PutFileQuery,
READ_FILE_STREAM_PATH, WALK_DIR_PATH, internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path,
put_file_stage_error_message, read_file_body_stream, verify_internode_rpc_signature, write_body_chunks_to_writer,
};
use bytes::Bytes;
use http::{HeaderMap, Method, StatusCode, Uri};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR,
};
use tokio::io;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio_stream::StreamExt;
use tokio_stream::iter;
rustfs/src/storage/rpc/lock.rs
+5 -2
View File
@@ -12,7 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use super::NodeService;
use rustfs_lock::LockRequest;
use rustfs_protos::proto_gen::node_service::*;
use tonic::{Request, Response, Status};
fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLockResult {
GenerallyLockResult {
@@ -269,7 +272,7 @@ impl NodeService {
#[cfg(test)]
mod tests {
use super::*;
use super::{lock_result_from_release, lock_result_from_response};
fn test_lock_id() -> rustfs_lock::LockId {
rustfs_lock::LockRequest::new(rustfs_lock::ObjectKey::new("bucket", "object"), rustfs_lock::LockType::Exclusive, "owner")
rustfs/src/storage/rpc/metrics.rs
+9 -1
View File
@@ -12,8 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use super::NodeService;
use crate::storage::rpc::encode_msgpack_map;
use crate::storage::storage_api::rpc_consumer::node_service::{CollectMetricsOpts, MetricType, collect_local_metrics};
use bytes::Bytes;
use rmp_serde::Deserializer;
use rustfs_protos::proto_gen::node_service::*;
use serde::Deserialize;
use std::io::Cursor;
use tonic::{Request, Response, Status};
use tracing::error;
impl NodeService {
pub(super) async fn handle_get_metrics(
rustfs/src/storage/rpc/mod.rs
+3 -3
View File
@@ -34,10 +34,10 @@ pub(crate) fn encode_msgpack_map<T: Serialize>(value: &T) -> Result<Vec<u8>, rmp
#[cfg(test)]
mod tests {
use super::*;
use rmp_serde::Deserializer;
use super::encode_msgpack_map;
use rmp_serde::{Deserializer, Serializer};
use rustfs_madmin::{BackendDisks, BackendInfo, Disk, ITEM_ONLINE, StorageInfo};
use serde::Deserialize;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::io::Cursor;
rustfs/src/storage/rpc/node_service.rs
+34 -34
View File
@@ -12,34 +12,27 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use super::super::{
CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, ECStore, Error, FileInfoVersions,
LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StoragePeerS3ClientExt as _,
UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, get_local_server_property,
load_bucket_metadata, reload_transition_tier_config, resolve_object_store_handle, set_bucket_metadata,
};
use crate::admin::service::{
config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot},
site_replication::reload_site_replication_runtime_state,
};
use crate::storage::contract::{
admin::StorageAdminApi,
bucket::{BucketOptions, DeleteBucketOptions, MakeBucketOptions},
#[cfg(test)]
use crate::storage::storage_api::rpc_consumer::node_service::STORAGE_CLASS_SUB_SYS;
#[cfg(test)]
use crate::storage::storage_api::rpc_consumer::node_service::{CollectMetricsOpts, MetricType};
use crate::storage::storage_api::rpc_consumer::node_service::{
DiskStore, ECStore, Error, LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, find_local_disk_by_ref,
reload_transition_tier_config, resolve_object_store_handle,
};
use crate::storage::runtime_sources;
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use bytes::Bytes;
use futures::Stream;
use futures_util::future::join_all;
use rmp_serde::Deserializer;
use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{FileInfo, MetacacheReader};
use rustfs_filemeta::MetacacheReader;
use rustfs_iam::store::UserType;
use rustfs_lock::{LockClient, LockRequest};
use rustfs_madmin::health::{
get_cpus, get_mem_info, get_os_info, get_partitions, get_proc_info, get_sys_config, get_sys_errors, get_sys_services,
};
use rustfs_madmin::net::get_net_info;
use rustfs_lock::LockClient;
use rustfs_protos::{
models::{PingBody, PingBodyBuilder},
proto_gen::node_service::{node_service_server::NodeService as Node, *},
@@ -127,11 +120,11 @@ fn unimplemented_rpc(method: &str) -> Status {
Status::unimplemented(format!("{method} is not implemented"))
}
fn background_rebalance_start_error_message(result: super::super::Result<()>) -> Option<String> {
fn background_rebalance_start_error_message(result: StorageResult<()>) -> Option<String> {
result.err().map(|err| format!("start_rebalance failed: {err}"))
}
fn stop_rebalance_response(result: super::super::Result<()>) -> StopRebalanceResponse {
fn stop_rebalance_response(result: StorageResult<()>) -> StopRebalanceResponse {
match result {
Ok(_) => StopRebalanceResponse {
success: true,
@@ -144,7 +137,7 @@ fn stop_rebalance_response(result: super::super::Result<()>) -> StopRebalanceRes
}
}
fn ensure_rpc_decommission_local_leader(store: &ECStore, idx: usize) -> super::super::Result<()> {
fn ensure_rpc_decommission_local_leader(store: &ECStore, idx: usize) -> StorageResult<()> {
let endpoints = store.endpoints();
let endpoint = endpoints
.as_ref()
@@ -1219,26 +1212,33 @@ impl Node for NodeService {
#[cfg(test)]
#[allow(unused_imports)]
mod tests {
use super::*;
use Request;
use super::{
CollectMetricsOpts, Error, MetricType, Node as _, NodeService, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS,
background_rebalance_start_error_message, make_server, stop_rebalance_response,
};
use bytes::Bytes;
use rustfs_protos::models::PingBodyBuilder;
use rustfs_protos::proto_gen::node_service::{
BackgroundHealStatusRequest, CheckPartsRequest, DeleteBucketMetadataRequest, DeleteBucketRequest, DeletePathsRequest,
DeletePolicyRequest, DeleteRequest, DeleteServiceAccountRequest, DeleteUserRequest, DeleteVersionRequest,
DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, DownloadProfileDataRequest, GenerallyLockRequest,
GetAllBucketStatsRequest, GetBucketInfoRequest, GetBucketStatsDataRequest, GetCpusRequest, GetMemInfoRequest,
GetMetacacheListingRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest,
GetSeLinuxInfoRequest, GetSrMetricsDataRequest, GetSysConfigRequest, GetSysErrorsRequest, HealBucketRequest,
ListBucketRequest, ListDirRequest, ListVolumesRequest, LoadBucketMetadataRequest, LoadGroupRequest,
GetMetacacheListingRequest, GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest,
GetProcInfoRequest, GetSeLinuxInfoRequest, GetSrMetricsDataRequest, GetSysConfigRequest, GetSysErrorsRequest,
HealBucketRequest, ListBucketRequest, ListDirRequest, ListVolumesRequest, LoadBucketMetadataRequest, LoadGroupRequest,
LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest,
MakeVolumesRequest, PingRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest,
ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest,
ServerInfoRequest, SignalServiceRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest,
UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
WriteRequest, node_service_client::NodeServiceClient, node_service_server::NodeServiceServer,
MakeVolumesRequest, Mss, PingRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest,
ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest,
RenamePartRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest, StatVolumeRequest,
StopRebalanceRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
WriteMetadataRequest, WriteRequest, node_service_client::NodeServiceClient, node_service_server::NodeServiceServer,
};
use std::collections::HashMap;
use tokio::net::TcpListener;
use tokio_stream::wrappers::TcpListenerStream;
use tonic::{Request, Response, Status};
fn create_test_node_service() -> NodeService {
make_server()
@@ -2527,7 +2527,7 @@ mod tests {
#[test]
fn test_background_rebalance_start_error_message_formats_error() {
let message = background_rebalance_start_error_message(Err(super::super::super::Error::other("boom")))
let message = background_rebalance_start_error_message(Err(Error::other("boom")))
.expect("background rebalance start failure should be formatted");
assert!(message.contains("start_rebalance failed"));
@@ -2536,7 +2536,7 @@ mod tests {
#[test]
fn test_stop_rebalance_response_reports_local_stop_error() {
let response = stop_rebalance_response(Err(super::super::super::Error::other("boom")));
let response = stop_rebalance_response(Err(Error::other("boom")));
assert!(!response.success);
assert!(response.error_info.as_deref().is_some_and(|message| message.contains("boom")));
@@ -2872,7 +2872,7 @@ mod tests {
let mut vars = HashMap::new();
vars.insert(PEER_RESTSIGNAL.to_string(), SERVICE_SIGNAL_RELOAD_DYNAMIC.to_string());
vars.insert(PEER_RESTSUB_SYS.to_string(), super::super::super::STORAGE_CLASS_SUB_SYS.to_string());
vars.insert(PEER_RESTSUB_SYS.to_string(), STORAGE_CLASS_SUB_SYS.to_string());
let request = Request::new(SignalServiceRequest {
vars: Some(Mss { value: vars }),
rustfs/src/storage/runtime_sources.rs
+1 -1
View File
@@ -14,7 +14,7 @@
use crate::config::RustFSBufferConfig;
use crate::runtime_sources as root_runtime_sources;
use crate::storage::ECStore;
use crate::storage::storage_api::runtime_sources_consumer::ECStore;
use rustfs_credentials::Credentials;
use rustfs_iam::{error::Result as IamResult, store::object::ObjectStore, sys::IamSys};
use rustfs_io_metrics::{PerformanceMetrics, internode_metrics::InternodeMetrics};
rustfs/src/storage/s3_api/bucket.rs
+6 -6
View File
@@ -12,12 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage::contract::{
use crate::storage::s3_api::common::rustfs_owner;
use crate::storage::storage_api::s3_api_consumer::bucket::contract::{
bucket::BucketInfo,
list::{ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info},
};
use crate::storage::s3_api::common::rustfs_owner;
use crate::storage::to_s3s_etag;
use crate::storage::storage_api::s3_api_consumer::bucket::to_s3s_etag;
use percent_encoding::percent_decode_str;
use s3s::dto::{
Bucket, CommonPrefix, DeleteMarkerEntry, EncodingType, ListBucketsOutput, ListObjectVersionsOutput, ListObjectsOutput,
@@ -27,7 +27,7 @@ use s3s::{S3Error, S3ErrorCode};
use tracing::debug;
use urlencoding::encode;
use crate::storage::StorageObjectInfo as ObjectInfo;
use crate::storage::storage_api::s3_api_consumer::bucket::StorageObjectInfo as ObjectInfo;
const S3_MAX_KEYS: i32 = 1000;
@@ -395,9 +395,9 @@ mod tests {
build_list_object_versions_output, build_list_objects_output, build_list_objects_v2_output,
parse_list_object_versions_params, parse_list_objects_v2_params,
};
use crate::storage::StorageObjectInfo as ObjectInfo;
use crate::storage::contract::bucket::BucketInfo;
use crate::storage::s3_api::common::rustfs_owner;
use crate::storage::storage_api::s3_api_consumer::bucket::StorageObjectInfo as ObjectInfo;
use crate::storage::storage_api::s3_api_consumer::bucket::contract::bucket::BucketInfo;
use s3s::S3ErrorCode;
use s3s::dto::{CommonPrefix, EncodingType, ListObjectsV2Output, Object};
use time::OffsetDateTime;
rustfs/src/storage/s3_api/multipart.rs
+6 -4
View File
@@ -12,9 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::storage::contract::multipart::{ListMultipartsInfo, ListPartsInfo};
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
use crate::storage::to_s3s_etag;
use crate::storage::storage_api::s3_api_consumer::multipart::contract::multipart::{ListMultipartsInfo, ListPartsInfo};
use crate::storage::storage_api::s3_api_consumer::multipart::to_s3s_etag;
use s3s::dto::{CommonPrefix, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp};
use s3s::{S3Error, S3ErrorCode};
@@ -195,9 +195,11 @@ mod tests {
MAX_MULTIPART_UPLOADS_LIST, build_list_multipart_uploads_output, build_list_parts_output,
parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
};
use crate::storage::contract::multipart::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo};
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
use crate::storage::to_s3s_etag;
use crate::storage::storage_api::s3_api_consumer::multipart::contract::multipart::{
ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo,
};
use crate::storage::storage_api::s3_api_consumer::multipart::to_s3s_etag;
use s3s::S3ErrorCode;
use s3s::dto::Timestamp;
use time::OffsetDateTime;
rustfs/src/storage/sse.rs
+31 -3
View File
@@ -70,7 +70,7 @@
//! ```
use super::StorageError;
use crate::storage::runtime_sources;
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
#[cfg(feature = "rio-v2")]
use aes_gcm::aead::Payload;
use aes_gcm::{
@@ -2402,10 +2402,38 @@ fn ssec_invalid_request(message: &str) -> ApiError {
}
#[cfg(test)]
#[allow(unused_imports)]
mod tests {
use super::*;
use http::HeaderValue;
#[cfg(feature = "rio-v2")]
use super::{
DARE_CIPHER_AES_256_GCM, DARE_CIPHER_CHACHA20_POLY1305, MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM, SEALED_KEY_IV_SIZE,
SEALED_KEY_SIZE, is_legacy_rustfs_managed_metadata, is_supported_sealed_object_key_cipher,
};
use super::{
DecryptionRequest, EncryptionKeyKind, EncryptionMaterial, EncryptionRequest, INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider,
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER,
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER,
PrepareEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER, SSEType, SseDekProvider, SsecParams, StorageError,
TestSseDekProvider, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, generate_ssec_nonce, is_managed_sse,
map_get_object_reader_error, mark_encrypted_multipart_metadata, normalize_managed_metadata, reset_sse_dek_provider,
resolve_effective_kms_key_id, sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata,
validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, validate_ssec_params,
verify_ssec_key_match,
};
use aes_gcm::aead::{Aead, KeyInit};
use aes_gcm::{Aes256Gcm, Key, Nonce};
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use http::{HeaderMap, HeaderValue};
use rustfs_kms::types::ObjectEncryptionContext;
use rustfs_rio::{DecryptReader, EncryptReader};
use rustfs_utils::http::headers::AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT;
use s3s::S3ErrorCode;
use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, ServerSideEncryption};
use std::collections::HashMap;
use std::sync::OnceLock;
use temp_env::async_with_vars;
use tokio::sync::Mutex;
rustfs/src/storage/storage_api.rs
+240
View File
@@ -60,6 +60,246 @@ pub(crate) type StorageObjectInfo = super::ObjectInfo;
pub(crate) type StorageObjectOptions = super::ObjectOptions;
pub(crate) type StorageObjectToDelete = contract::object::ObjectToDelete;
pub(crate) type StoragePutObjReader = super::PutObjReader;
pub(crate) use super::ecfs_extend::{
RFC1123, apply_bucket_default_lock_retention, apply_cors_headers, check_preconditions, get_buffer_size_opt_in,
get_validated_store, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention,
parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
remove_object_lock_metadata_for_copy, validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter,
validate_object_key, wrap_response_with_cors,
};
pub(crate) use super::sse::{
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, extract_server_side_encryption_from_headers, sse_decryption,
sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read,
validate_sse_headers_for_write, validate_ssec_for_read,
};
pub(crate) mod access_consumer {
pub(crate) use super::super::access::{
PostObjectRequestMarker, ReqInfo, authorize_request, has_bypass_governance_header, req_info_mut, req_info_ref,
};
pub(crate) mod contract {
pub(crate) mod bucket {
pub(crate) use super::super::super::contract::bucket::BucketOperations;
}
}
}
pub(crate) mod concurrency_consumer {
pub(crate) use super::super::concurrency::{
ConcurrencyManager, GetObjectGuard, IoQueueStatus, IoStrategy, PutObjectGuard, get_concurrency_aware_buffer_size,
get_concurrency_manager, get_put_concurrency_aware_buffer_size,
};
}
pub(crate) mod deadlock_detector_consumer {
#[cfg(test)]
pub(crate) use super::super::deadlock_detector::RequestHangDetectionPolicy;
pub(crate) use super::super::deadlock_detector::{DeadlockDetector, get_deadlock_detector};
}
pub(crate) mod ecfs_consumer {
pub(crate) mod contract {
pub(crate) mod bucket {
pub(crate) use super::super::super::contract::bucket::{BucketOperations, BucketOptions};
}
pub(crate) mod object {
pub(crate) use super::super::super::contract::object::{ObjectLockRetentionOptions, ObjectOperations};
}
}
pub(crate) mod object_lock {
pub(crate) use super::super::super::{
parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled,
};
}
pub(crate) type StorageObjectOptions = super::StorageObjectOptions;
}
pub(crate) mod ecfs_extend_consumer {
pub(crate) mod contract {
pub(crate) mod bucket {
pub(crate) use super::super::super::contract::bucket::{BucketOperations, BucketOptions};
}
pub(crate) mod object {
pub(crate) use super::super::super::contract::object::ObjectToDelete;
}
}
pub(crate) type StorageObjectInfo = super::StorageObjectInfo;
}
pub(crate) mod head_prefix_consumer {
pub(crate) use super::super::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children};
pub(crate) mod contract {
pub(crate) mod list {
pub(crate) use super::super::super::contract::list::ListOperations;
}
}
}
pub(crate) mod helper_consumer {
pub(crate) use super::super::helper::{OperationHelper, spawn_background_with_context};
pub(crate) type StorageObjectInfo = super::StorageObjectInfo;
}
pub(crate) mod options_consumer {
pub(crate) use super::super::options::{
copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime,
extract_metadata_from_mime_with_object_name, filter_object_metadata, get_complete_multipart_upload_opts,
get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, parse_copy_source_range, put_opts,
validate_archive_content_encoding,
};
pub(crate) mod contract {
pub(crate) mod object {
pub(crate) use super::super::super::contract::object::HTTPPreconditions;
}
pub(crate) mod range {
pub(crate) use super::super::super::contract::range::HTTPRangeSpec;
}
}
pub(crate) type StorageObjectOptions = super::StorageObjectOptions;
}
pub(crate) mod request_context_consumer {
pub(crate) use super::super::request_context::{
RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced,
};
}
pub(crate) mod rpc_consumer {
pub(crate) use super::super::rpc::InternodeRpcService;
pub(crate) mod http_service {
pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = super::super::DEFAULT_READ_BUFFER_SIZE;
pub(crate) use super::super::{StorageDiskRpcExt, WalkDirOptions, find_local_disk_by_ref, verify_rpc_signature};
}
pub(crate) mod node_service {
pub(crate) use super::super::{
CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, ECStore, Error, FileInfoVersions,
LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt, StoragePeerS3ClientExt,
UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, get_local_server_property,
load_bucket_metadata, reload_transition_tier_config, resolve_object_store_handle, set_bucket_metadata,
};
pub(crate) type StorageResult<T> = super::super::Result<T>;
#[cfg(test)]
pub(crate) const STORAGE_CLASS_SUB_SYS: &str = super::super::STORAGE_CLASS_SUB_SYS;
pub(crate) mod contract {
pub(crate) mod admin {
pub(crate) use super::super::super::super::contract::admin::StorageAdminApi;
}
pub(crate) mod bucket {
pub(crate) use super::super::super::super::contract::bucket::{
BucketOptions, DeleteBucketOptions, MakeBucketOptions,
};
}
}
}
}
pub(crate) mod runtime_sources_consumer {
pub(crate) type ECStore = super::ECStore;
pub(crate) use crate::storage::runtime_sources;
}
pub(crate) mod s3_api_consumer {
pub(crate) mod bucket {
pub(crate) use super::super::super::s3_api::bucket::{
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
build_list_objects_output, build_list_objects_v2_output, parse_list_object_versions_params,
parse_list_objects_v2_params,
};
pub(crate) mod contract {
pub(crate) mod bucket {
pub(crate) use super::super::super::super::contract::bucket::BucketInfo;
}
pub(crate) mod list {
pub(crate) use super::super::super::super::contract::list::{ListObjectVersionsInfo, ListObjectsV2Info};
}
}
pub(crate) type StorageObjectInfo = super::super::StorageObjectInfo;
pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag {
super::super::to_s3s_etag(etag)
}
}
pub(crate) mod common {
pub(crate) use super::super::super::s3_api::common::rustfs_owner;
}
pub(crate) mod multipart {
pub(crate) use super::super::super::s3_api::multipart::{
ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output,
parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
};
pub(crate) mod contract {
pub(crate) mod multipart {
pub(crate) use super::super::super::super::contract::multipart::{ListMultipartsInfo, ListPartsInfo};
#[cfg(test)]
pub(crate) use super::super::super::super::contract::multipart::{MultipartInfo, PartInfo};
}
}
pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag {
super::super::to_s3s_etag(etag)
}
}
}
pub(crate) mod sse_consumer {
pub(crate) use super::super::sse::{
EncryptionKeyKind, SSEType, build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers,
extract_ssekms_context_from_headers, map_get_object_reader_error, mark_encrypted_multipart_metadata,
};
pub(crate) use super::{
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, apply_bucket_default_lock_retention,
extract_server_side_encryption_from_headers, get_buffer_size_opt_in, sse_decryption, sse_encryption,
sse_prepare_encryption,
};
}
pub(crate) mod timeout_wrapper_consumer {
pub(crate) use super::super::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper};
}
pub(crate) mod tonic_service_consumer {
pub(crate) use super::super::tonic_service::make_server;
}
#[cfg(test)]
pub(crate) mod test_consumer {
pub(crate) use super::super::{
apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions, decode_tags_to_map,
get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal, matches_origin_pattern, parse_etag,
parse_object_lock_legal_hold, parse_object_lock_retention, process_lambda_configurations, process_queue_configurations,
process_topic_configurations, remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata,
validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter,
};
pub(crate) use super::{
BucketMetadata, DEFAULT_READ_BUFFER_SIZE, bucket_metadata_sys_initialized, get_global_bucket_metadata_sys,
set_bucket_metadata,
};
pub(crate) type StorageObjectInfo = super::StorageObjectInfo;
}
pub(crate) mod ecstore_admin {
pub(crate) use rustfs_ecstore::api::admin::{get_local_server_property, get_server_info};
rustfs/src/storage/timeout_wrapper.rs
+4 -1
View File
@@ -462,8 +462,11 @@ pub fn get_io_buffer_size() -> usize {
}
#[cfg(test)]
#[allow(unused_imports)]
mod tests {
use super::*;
use super::{
GetObjectTimeoutPolicy, RequestTimeoutWrapper, TimedGetObjectResult, get_duplex_buffer_size, get_io_buffer_size,
};
use std::time::Duration;
#[test]
rustfs/src/storage_api.rs
+197 -77
View File
@@ -18,24 +18,39 @@
use rustfs_storage_api as storage_contracts;
pub(crate) mod capacity {
pub(crate) use crate::storage::{all_local_disk, disk_drive_path, disk_endpoint};
pub(crate) mod service {
pub(crate) use crate::storage::storage_api::{all_local_disk, disk_drive_path, disk_endpoint};
}
}
pub(crate) mod cluster {
pub(crate) use super::storage_contracts::{
CapabilitySnapshotError, CapabilityStatus, DiskCapabilities, MemorySamplingState, ObservabilitySnapshot,
ObservabilitySnapshotProvider, PlatformSupport, TopologyCapabilities, TopologySnapshot, TopologySnapshotProvider,
UserspaceProfilingCapability,
};
pub(crate) use crate::storage::{EndpointServerPools, topology_snapshot_from_endpoint_pools_with_capabilities};
pub(crate) mod contract {
pub(crate) mod capability {
#[cfg(test)]
pub(crate) use super::super::super::storage_contracts::CapabilityState;
pub(crate) use super::super::super::storage_contracts::{
CapabilitySnapshotError, CapabilityStatus, MemorySamplingState, PlatformSupport, UserspaceProfilingCapability,
};
}
pub(crate) mod observability {
pub(crate) use super::super::super::storage_contracts::{ObservabilitySnapshot, ObservabilitySnapshotProvider};
}
pub(crate) mod topology {
pub(crate) use super::super::super::storage_contracts::{
DiskCapabilities, TopologyCapabilities, TopologySnapshot, TopologySnapshotProvider,
};
}
}
pub(crate) use crate::storage::storage_api::{EndpointServerPools, topology_snapshot_from_endpoint_pools_with_capabilities};
#[cfg(test)]
pub(crate) use super::storage_contracts::CapabilityState;
#[cfg(test)]
pub(crate) use crate::storage::{Endpoint, Endpoints, PoolEndpoints};
pub(crate) use crate::storage::storage_api::{Endpoint, Endpoints, PoolEndpoints};
pub(crate) mod control_plane {
pub(crate) use crate::storage::ecstore_cluster::{
pub(crate) use crate::storage::storage_api::ecstore_cluster::{
ClusterControlPlane, ClusterControlPlaneSnapshot, ClusterLocalNodeStorageSnapshot, ClusterMembershipSnapshot,
ClusterPeerHealthSnapshot, ClusterPoolStateSnapshot,
};
@@ -44,103 +59,208 @@ pub(crate) mod cluster {
#[cfg(test)]
pub(crate) mod config_test {
pub(crate) use crate::storage::DisksLayout;
pub(crate) use crate::storage::storage_api::DisksLayout;
}
pub(crate) mod error {
pub(crate) use super::storage_contracts::HTTPRangeError;
pub(crate) use crate::storage::{QuotaError, StorageError};
pub(crate) mod contract {
pub(crate) mod range {
pub(crate) use super::super::super::storage_contracts::HTTPRangeError;
}
}
pub(crate) use crate::storage::storage_api::{QuotaError, StorageError};
}
pub(crate) mod protocols {
pub(crate) mod access {
pub(crate) use crate::storage::access::ReqInfo;
}
pub(crate) mod ecfs {
pub(crate) type FS = crate::storage::FS;
}
pub(crate) mod request_context {
pub(crate) use crate::storage::request_context::RequestContext;
pub(crate) mod client {
pub(crate) use crate::storage::storage_api::access_consumer::ReqInfo;
pub(crate) type FS = crate::storage::storage_api::FS;
pub(crate) use crate::storage::storage_api::request_context_consumer::RequestContext;
}
}
pub(crate) mod server {
pub(crate) use super::storage_contracts::StorageAdminApi;
pub(crate) use crate::storage::{
ECStore, Endpoint, EndpointServerPools, Error, EventArgs, StorageObjectInfo, TONIC_RPC_PREFIX, apply_cors_headers,
is_dist_erasure, read_config, register_event_dispatch_hook, save_config, verify_rpc_signature,
};
pub(crate) mod event {
pub(crate) mod contract {
#[cfg(test)]
pub(crate) mod lifecycle {
pub(crate) use super::super::super::super::storage_contracts::TransitionedObject;
}
}
#[cfg(test)]
pub(crate) use super::storage_contracts::TransitionedObject;
#[cfg(test)]
pub(crate) use crate::storage::{Endpoints, PoolEndpoints};
pub(crate) mod ecfs {
pub(crate) type FS = crate::storage::FS;
pub(crate) use crate::storage::storage_api::{EventArgs, StorageObjectInfo, register_event_dispatch_hook};
}
pub(crate) mod request_context {
pub(crate) use crate::storage::request_context::{
RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced,
};
pub(crate) mod http {
pub(crate) use crate::storage::storage_api::{TONIC_RPC_PREFIX, verify_rpc_signature};
pub(crate) mod ecfs {
pub(crate) type FS = crate::storage::storage_api::FS;
}
pub(crate) mod request_context {
pub(crate) use crate::storage::storage_api::request_context_consumer::{
RequestContext, extract_request_id_from_headers,
};
}
pub(crate) mod rpc {
pub(crate) use crate::storage::storage_api::rpc_consumer::InternodeRpcService;
}
pub(crate) mod tonic_service {
pub(crate) use crate::storage::storage_api::tonic_service_consumer::make_server;
}
}
pub(crate) mod rpc {
pub(crate) use crate::storage::rpc::InternodeRpcService;
pub(crate) mod layer {
pub(crate) use crate::storage::storage_api::apply_cors_headers;
pub(crate) mod request_context {
pub(crate) use crate::storage::storage_api::request_context_consumer::{
RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced,
};
}
}
pub(crate) mod tonic_service {
pub(crate) use crate::storage::tonic_service::make_server;
pub(crate) mod module_switch {
pub(crate) use crate::storage::storage_api::{Error, read_config, save_config};
}
pub(crate) mod readiness {
pub(crate) mod contract {
pub(crate) mod admin {
pub(crate) use super::super::super::super::storage_contracts::StorageAdminApi;
}
}
pub(crate) use crate::storage::storage_api::{Endpoint, EndpointServerPools, is_dist_erasure};
#[cfg(test)]
pub(crate) use crate::storage::storage_api::{Endpoints, PoolEndpoints};
}
pub(crate) mod runtime_sources {
pub(crate) use crate::storage::storage_api::{ECStore, EndpointServerPools};
}
}
pub(crate) mod startup {
pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions};
pub(crate) use crate::storage::{
DynReplicationPool, ECStore, EndpointServerPools, Result, get_bucket_notification_config, init_background_replication,
init_bucket_metadata_sys, init_ecstore_config, init_global_config_sys, init_local_disks, init_lock_clients,
new_global_notification_sys, prewarm_local_disk_id_map, process_lambda_configurations, process_queue_configurations,
process_topic_configurations, set_global_endpoints, set_global_region, set_global_rustfs_port,
set_workload_admission_snapshot_provider, shutdown_background_services, try_migrate_bucket_metadata,
try_migrate_iam_config, try_migrate_server_config, update_erasure_type,
};
pub(crate) mod background {
pub(crate) use crate::storage::storage_api::{ECStore, set_workload_admission_snapshot_provider};
}
#[cfg(test)]
pub(crate) use crate::storage::Error;
pub(crate) mod bucket_metadata {
pub(crate) mod contract {
pub(crate) mod bucket {
pub(crate) use super::super::super::super::storage_contracts::{BucketOperations, BucketOptions};
}
}
pub(crate) use crate::storage::storage_api::{
ECStore, init_bucket_metadata_sys, try_migrate_bucket_metadata, try_migrate_iam_config,
};
}
pub(crate) mod deadlock {
pub(crate) use crate::storage::storage_api::deadlock_detector_consumer::get_deadlock_detector;
}
pub(crate) mod fs_guard {
pub(crate) use crate::storage::storage_api::EndpointServerPools;
}
pub(crate) mod iam {
pub(crate) use crate::storage::storage_api::ECStore;
}
pub(crate) mod init {
pub(crate) use crate::storage::storage_api::{
get_bucket_notification_config, process_lambda_configurations, process_queue_configurations,
process_topic_configurations,
};
pub(crate) mod concurrency {
pub(crate) use crate::storage::storage_api::concurrency_consumer::get_concurrency_manager;
}
pub(crate) mod ecfs {
pub(crate) type FS = crate::storage::storage_api::FS;
}
}
pub(crate) mod lifecycle {
pub(crate) use crate::storage::storage_api::ECStore;
}
pub(crate) mod notification {
pub(crate) use crate::storage::storage_api::{EndpointServerPools, Result, new_global_notification_sys};
#[cfg(test)]
pub(crate) use crate::storage::storage_api::Error;
}
pub(crate) mod runtime_sources {
pub(crate) use crate::storage::storage_api::{DynReplicationPool, set_global_region, set_global_rustfs_port};
}
pub(crate) mod services {
pub(crate) use crate::storage::storage_api::{ECStore, EndpointServerPools};
}
pub(crate) mod shutdown {
pub(crate) use crate::storage::storage_api::shutdown_background_services;
}
pub(crate) mod storage {
pub(crate) use crate::storage::storage_api::{
ECStore, EndpointServerPools, init_background_replication, init_ecstore_config, init_global_config_sys,
init_local_disks, init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints, try_migrate_server_config,
update_erasure_type,
};
}
}
pub(crate) mod workload {
pub(crate) mod admission {
pub(crate) use crate::storage::storage_api::{bucket_metadata_runtime_initialized, replication_queue_current_count};
}
pub(crate) mod concurrency {
pub(crate) use crate::storage::concurrency::get_concurrency_manager;
}
pub(crate) mod deadlock_detector {
pub(crate) use crate::storage::deadlock_detector::get_deadlock_detector;
}
pub(crate) mod ecfs {
pub(crate) type FS = crate::storage::FS;
pub(crate) use crate::storage::storage_api::concurrency_consumer::get_concurrency_manager;
}
}
pub(crate) mod table {
pub(crate) use super::storage_contracts::{
HTTPPreconditions, HTTPRangeSpec, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, NamespaceLocking, ObjectIO,
ObjectInfoOrErr, ObjectOperations, WalkOptions,
};
pub(crate) use crate::storage::{
pub(crate) mod contract {
pub(crate) mod http {
pub(crate) use super::super::super::storage_contracts::HTTPPreconditions;
}
pub(crate) mod list {
pub(crate) use super::super::super::storage_contracts::{
ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr, WalkOptions,
};
}
pub(crate) mod namespace {
pub(crate) use super::super::super::storage_contracts::NamespaceLocking;
}
pub(crate) mod object {
pub(crate) use super::super::super::storage_contracts::{ObjectIO, ObjectOperations};
}
pub(crate) mod range {
pub(crate) use super::super::super::storage_contracts::HTTPRangeSpec;
}
}
pub(crate) use crate::storage::storage_api::{
BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG,
BUCKET_TABLE_RESERVED_PREFIX, Error, RUSTFS_META_BUCKET, StorageDeletedObject, StorageError, StorageGetObjectReader,
StorageObjectInfo, StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, get_bucket_metadata,
get_lock_acquire_timeout, table_catalog_path_hash,
};
}
pub(crate) mod workload {
pub(crate) use crate::storage::{bucket_metadata_runtime_initialized, replication_queue_current_count};
pub(crate) mod concurrency {
pub(crate) use crate::storage::concurrency::get_concurrency_manager;
}
}
rustfs/src/table_catalog.rs
+10 -5
View File
@@ -27,13 +27,18 @@ use std::{
time::{Duration as StdDuration, Instant},
};
use crate::storage_api::table::contract::http::HTTPPreconditions;
use crate::storage_api::table::contract::list::{
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
ListOperations as StorageListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
};
use crate::storage_api::table::contract::namespace::NamespaceLocking as StorageNamespaceLocking;
use crate::storage_api::table::contract::object::{ObjectIO as StorageObjectIO, ObjectOperations as StorageObjectOperations};
use crate::storage_api::table::contract::range::HTTPRangeSpec;
use crate::storage_api::table::{
BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG,
BUCKET_TABLE_RESERVED_PREFIX, Error as EcstoreError, HTTPPreconditions, HTTPRangeSpec,
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
ListOperations as StorageListOperations, NamespaceLocking as StorageNamespaceLocking, ObjectIO as StorageObjectIO,
ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations as StorageObjectOperations, RUSTFS_META_BUCKET, StorageError,
WalkOptions as StorageWalkOptions, get_bucket_metadata, get_lock_acquire_timeout, table_catalog_path_hash,
BUCKET_TABLE_RESERVED_PREFIX, Error as EcstoreError, RUSTFS_META_BUCKET, StorageError, get_bucket_metadata,
get_lock_acquire_timeout, table_catalog_path_hash,
};
use bytes::Bytes;
use datafusion::{
rustfs/src/workload_admission.rs
+1 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::runtime_sources::resolve_replication_pool_handle;
use crate::storage_api::workload::{bucket_metadata_runtime_initialized, replication_queue_current_count};
use crate::storage_api::workload::admission::{bucket_metadata_runtime_initialized, replication_queue_current_count};
use rustfs_concurrency::{
AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider,
WorkloadClass,