mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
refactor: route app and admin storage helpers (#3863)
* refactor: route admin storage helper imports * refactor: route app storage io helper imports
This commit is contained in:
Zhengchao An
@@ -5,9 +5,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
## Current Context
|
||||
|
||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||
- Branch: `overtrue/arch-app-storage-boundary-imports`
|
||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211`.
|
||||
- Based on: stacked on API-210 after PR #3853 merged.
|
||||
- Branch: `overtrue/arch-app-storage-io-helper-boundary`
|
||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213`.
|
||||
- Based on: stacked on API-212 after PR #3854 merged.
|
||||
- PR type for this branch: `consumer-migration`
|
||||
- Runtime behavior changes: none.
|
||||
- Rust code changes: route replication pool, outbound TLS generation, runtime
|
||||
@@ -45,7 +45,7 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
readiness-time, observability, metrics, buffer, and TLS runtime source
|
||||
helpers, plus server readiness/audit/event/module-switch runtime source
|
||||
helpers, storage request/RPC/SSE runtime source helpers, and admin
|
||||
handler/service/router runtime source helpers, plus root auth/init/config/protocol/workload, app usecase, storage node-service, remaining admin grouped context import runtime source helpers, app bucket/object/multipart usecase explicit storage imports, app select/bucket/object/multipart explicit S3 DTO plus ECFS owner imports, app-local S3 API response helper imports for bucket/object/multipart usecases, and app-local storage helper boundary imports for select/bucket/object/multipart usecases,
|
||||
handler/service/router runtime source helpers, plus root auth/init/config/protocol/workload, app usecase, storage node-service, remaining admin grouped context import runtime source helpers, app bucket/object/multipart usecase explicit storage imports, app select/bucket/object/multipart explicit S3 DTO plus ECFS owner imports, app-local S3 API response helper imports for bucket/object/multipart usecases, app-local storage helper boundary imports for select/bucket/object/multipart usecases, admin/app test storage helper boundary imports for request context, authorization, object options, and ECFS test harness consumers, and app storage IO/compression/set-disk helper boundary imports for object and multipart usecases,
|
||||
through AppContext-first or owner-crate resolver boundaries.
|
||||
- CI/script changes: lock completed owner and test/fuzz boundaries against
|
||||
bare/glob imports, scattered raw ECStore facade subpaths, and startup
|
||||
@@ -56,8 +56,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
event-bridge thin module regressions, plus IAM runtime-source bypasses;
|
||||
accept the reviewed AppContext resolver reverse dependencies in the layer
|
||||
baseline, and block direct admin AppContext resolver consumers outside the
|
||||
admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, and reject direct storage helper imports from app select/usecase files.
|
||||
- Docs changes: record the API-136 through API-211 owner facade and lifecycle
|
||||
admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, reject direct storage helper imports from app select/usecase files, reject completed app/admin storage helper bypasses, and reject app usecase bypasses for migrated storage IO/compression/set-disk helpers.
|
||||
- Docs changes: record the API-136 through API-213 owner facade and lifecycle
|
||||
runtime-source cleanup.
|
||||
|
||||
## Phase 0 Tasks
|
||||
@@ -5041,14 +5041,48 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
layer guards, diff hygiene, residual direct storage helper scan, Rust risk
|
||||
scan, fast PR gate, and full PR gate before PR.
|
||||
|
||||
- [x] `API-212` Route app/admin storage helper imports through local boundaries.
|
||||
- Do: add an admin-local storage helper boundary, route admin router,
|
||||
console, heal, metrics, and object-zip handlers through it, and route app
|
||||
integration tests through the existing app storage boundary.
|
||||
- Acceptance: the completed app/admin consumers no longer import migrated
|
||||
storage helper APIs directly, and migration rules reject direct bypasses.
|
||||
- Must preserve: admin authorization and request tracing, console request
|
||||
context propagation, admin object zip storage options, and app lifecycle and
|
||||
capacity dirty-scope test harness behavior.
|
||||
- Verification: focused RustFS admin/app compile/tests, formatting, migration
|
||||
and layer guards, diff hygiene, residual direct storage helper scan, Rust
|
||||
risk scan, fast PR gate, and full PR gate before PR.
|
||||
|
||||
- [x] `API-213` Route app storage IO helper imports through app boundary.
|
||||
- Do: extend the app-local storage helper boundary for compression, reader,
|
||||
write-plan, and set-disk helpers, then route object and multipart usecases
|
||||
through it instead of app-level compatibility aliases.
|
||||
- Acceptance: app object and multipart usecases no longer consume the
|
||||
migrated storage IO/compression/set-disk helpers through `super::` aliases,
|
||||
the app root no longer re-exports those helpers, and migration rules reject
|
||||
reintroducing the completed bypasses.
|
||||
- Must preserve: object compression metadata, hash-reader checksum handling,
|
||||
multipart SSE-C/SSE-S3 write-encryption planning, valid-storage-class
|
||||
checks, lock-acquire timeout reads, and test-only reader wrappers.
|
||||
- Verification: focused RustFS app compile/tests, formatting, migration and
|
||||
layer guards, diff hygiene, residual migrated helper scan, Rust risk scan,
|
||||
fast PR gate, and full PR gate before PR.
|
||||
|
||||
## Next PRs
|
||||
|
||||
1. `consumer-migration`: continue reducing direct global reads behind AppContext resolver boundaries.
|
||||
1. `consumer-migration`: batch remaining app/admin/storage helper boundary and owner facade cleanup.
|
||||
|
||||
## Pre-Push Review Log
|
||||
|
||||
| Expert | Status | Notes |
|
||||
|---|---|---|
|
||||
| Quality/architecture | pass | API-213 removes app root IO/compression/set-disk compatibility aliases and routes object/multipart usecases through the app storage_api boundary. |
|
||||
| Migration preservation | pass | Object compression metadata, hash-reader checksum handling, multipart write-encryption planning, storage-class checks, lock timeouts, and test reader wrappers keep the same storage implementations. |
|
||||
| Testing/verification | pass | Focused app compile/tests, formatting, migration/layer guards, residual migrated helper scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. |
|
||||
| Quality/architecture | pass | API-212 routes completed app/admin storage helper consumers through local storage_api boundaries and adds a direct-bypass guard. |
|
||||
| Migration preservation | pass | Admin authorization/tracing, console request context propagation, object-zip storage options, and app lifecycle/capacity test harness behavior keep the same storage implementations. |
|
||||
| Testing/verification | pass | Focused admin/app compile/tests, formatting, migration/layer guards, residual direct storage helper scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. |
|
||||
| Quality/architecture | pass | API-211 routes app select/usecase storage helper imports through an app-local storage_api boundary and adds a direct-bypass guard. |
|
||||
| Migration preservation | pass | Select preflight, bucket authorization/background work, multipart SSE/options parsing, and object SSE/concurrency/prefix/timeout paths keep the same storage helper implementations. |
|
||||
| Testing/verification | pass | Focused RustFS app compile/tests, formatting, migration/layer guards, residual direct storage helper scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. |
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
use crate::admin::handlers::health::{HealthProbe, build_health_response_parts, collect_dependency_readiness};
|
||||
use crate::admin::runtime_sources::resolve_oidc_handle;
|
||||
use crate::admin::storage_api::RequestContext;
|
||||
use crate::app::admin_usecase::DefaultAdminUsecase;
|
||||
use crate::license::has_valid_license;
|
||||
use crate::server::has_path_prefix;
|
||||
@@ -21,7 +22,6 @@ use crate::server::{
|
||||
CONSOLE_PREFIX, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HeaderMapCarrier, LICENSE, RUSTFS_ADMIN_PREFIX,
|
||||
RequestContextLayer, VERSION,
|
||||
};
|
||||
use crate::storage::request_context::RequestContext;
|
||||
use crate::version::build;
|
||||
use axum::{
|
||||
Json, Router,
|
||||
|
||||
@@ -17,9 +17,9 @@ use super::super::is_reserved_or_invalid_bucket;
|
||||
use crate::admin::auth::{authenticate_request, validate_admin_request};
|
||||
use crate::admin::router::{AdminOperation, Operation, S3Router};
|
||||
use crate::admin::runtime_sources::resolve_object_store_handle;
|
||||
use crate::admin::storage_api::spawn_traced;
|
||||
use crate::server::ADMIN_PREFIX;
|
||||
use crate::server::RemoteAddr;
|
||||
use crate::storage::request_context::spawn_traced;
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderValue, Uri};
|
||||
use hyper::{Method, StatusCode};
|
||||
|
||||
@@ -21,9 +21,9 @@
|
||||
use super::super::{CollectMetricsOpts, MetricType, collect_local_metrics};
|
||||
use crate::admin::auth::validate_admin_request;
|
||||
use crate::admin::router::Operation;
|
||||
use crate::admin::storage_api::spawn_traced;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::server::RemoteAddr;
|
||||
use crate::storage::request_context::spawn_traced;
|
||||
use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt};
|
||||
use http::{HeaderMap, HeaderValue, Uri};
|
||||
|
||||
@@ -14,11 +14,11 @@
|
||||
|
||||
use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, Operation, S3Router};
|
||||
use crate::admin::runtime_sources::{resolve_action_credentials, resolve_object_store_handle, resolve_region};
|
||||
use crate::admin::storage_api::{ReqInfo, StorageObjectOptions as ObjectOptions, authorize_request};
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::error::ApiError;
|
||||
use crate::license::license_check;
|
||||
use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||
use crate::storage::access::{ReqInfo, authorize_request};
|
||||
use aes_gcm::{
|
||||
Aes256Gcm, Key, Nonce,
|
||||
aead::{Aead, KeyInit},
|
||||
@@ -47,8 +47,6 @@ use tokio_util::io::ReaderStream;
|
||||
use url::form_urlencoded;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::storage::StorageObjectOptions as ObjectOptions;
|
||||
|
||||
const OBJECT_ZIP_DOWNLOAD_TOKEN_TTL: Duration = Duration::minutes(5);
|
||||
const ZIP_STREAM_BUFFER_SIZE: usize = 1024 * 1024;
|
||||
const ZIP_OBJECT_BUFFER_SIZE: usize = 128 * 1024;
|
||||
|
||||
@@ -24,6 +24,7 @@ pub mod router;
|
||||
pub(crate) mod runtime_sources;
|
||||
pub mod service;
|
||||
pub mod site_replication_identity;
|
||||
pub(crate) mod storage_api;
|
||||
pub mod utils;
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -29,6 +29,7 @@ use crate::admin::runtime_sources::{
|
||||
resolve_object_store_handle, resolve_region, resolve_replication_pool_handle, resolve_replication_stats_handle,
|
||||
resolve_server_config,
|
||||
};
|
||||
use crate::admin::storage_api::{ReqInfo, authorize_request, spawn_traced};
|
||||
use crate::app::object_usecase::DefaultObjectUsecase;
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::error::ApiError;
|
||||
@@ -36,8 +37,6 @@ use crate::license::license_check;
|
||||
use crate::server::{
|
||||
ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, is_admin_path,
|
||||
};
|
||||
use crate::storage::access::{ReqInfo, authorize_request};
|
||||
use crate::storage::request_context::spawn_traced;
|
||||
use aws_sdk_s3::primitives::ByteStream as AwsByteStream;
|
||||
use bytes::Bytes;
|
||||
use futures::{Stream, StreamExt};
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use crate::storage::StorageObjectOptions;
|
||||
pub(crate) use crate::storage::access::{ReqInfo, authorize_request};
|
||||
pub(crate) use crate::storage::request_context::{RequestContext, spawn_traced};
|
||||
@@ -29,7 +29,7 @@ use tokio::fs;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::storage::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader};
|
||||
use super::storage_api::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader};
|
||||
|
||||
static CAPACITY_DIRTY_SCOPE_ENV: OnceLock<(Vec<PathBuf>, Arc<ECStore>, TempDir)> = OnceLock::new();
|
||||
static CAPACITY_DIRTY_SCOPE_INIT: Once = Once::new();
|
||||
|
||||
@@ -12,6 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::storage_api::ecfs::FS;
|
||||
use super::storage_api::{
|
||||
StorageObjectInfo as ObjectInfo, StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader,
|
||||
};
|
||||
use super::{
|
||||
AppWarmBackend, ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, TierConfig, TierType, WarmBackendGetOpts,
|
||||
metadata::{BUCKET_LIFECYCLE_CONFIG, OBJECT_LOCK_CONFIG},
|
||||
@@ -22,10 +26,6 @@ use super::{
|
||||
use super::{multipart_usecase::DefaultMultipartUsecase, object_usecase::DefaultObjectUsecase};
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::app::runtime_sources::resolve_tier_config_handle;
|
||||
use crate::storage::ecfs::FS;
|
||||
use crate::storage::{
|
||||
StorageObjectInfo as ObjectInfo, StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::FutureExt;
|
||||
use futures::stream;
|
||||
|
||||
@@ -68,10 +68,6 @@ mod ecstore_client {
|
||||
pub(crate) use crate::storage::ecstore_client::{object_api_utils, transition_api};
|
||||
}
|
||||
|
||||
mod ecstore_compression {
|
||||
pub(crate) use crate::storage::ecstore_compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||
}
|
||||
|
||||
mod ecstore_config {
|
||||
pub(crate) use crate::storage::ecstore_config::storageclass;
|
||||
}
|
||||
@@ -88,15 +84,11 @@ mod ecstore_tier {
|
||||
pub(crate) use crate::storage::ecstore_tier::{tier, tier_config, warm_backend};
|
||||
}
|
||||
|
||||
pub(crate) const MIN_DISK_COMPRESSIBLE_SIZE: usize = ecstore_compression::MIN_DISK_COMPRESSIBLE_SIZE;
|
||||
|
||||
pub(crate) type DiskError = crate::storage::DiskError;
|
||||
pub(crate) type DynReader = crate::storage::DynReader;
|
||||
pub(crate) type DynReplicationPool = crate::storage::DynReplicationPool;
|
||||
pub(crate) type ECStore = crate::storage::ECStore;
|
||||
pub(crate) type EndpointServerPools = crate::storage::EndpointServerPools;
|
||||
pub(crate) type ExpiryState = crate::storage::ExpiryState;
|
||||
pub(crate) type HashReader = crate::storage::HashReader;
|
||||
pub(crate) type NotificationSys = crate::storage::NotificationSys;
|
||||
pub(crate) type BucketBandwidthMonitor = crate::storage::BucketBandwidthMonitor;
|
||||
pub(crate) type ObjectStoreResolver = crate::storage::ObjectStoreResolver;
|
||||
@@ -108,20 +100,12 @@ pub(crate) type RebalStatus = crate::storage::ecstore_rebalance::RebalStatus;
|
||||
pub(crate) type StorageError = crate::storage::StorageError;
|
||||
pub(crate) type Error = StorageError;
|
||||
pub(crate) type TierConfigMgr = crate::storage::TierConfigMgr;
|
||||
pub(crate) type WriteEncryption = crate::storage::WriteEncryption;
|
||||
pub(crate) type WritePlan = crate::storage::WritePlan;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) type DecryptReader<R> = crate::storage::DecryptReader<R>;
|
||||
#[cfg(test)]
|
||||
pub(crate) type EncryptReader<R> = crate::storage::EncryptReader<R>;
|
||||
#[cfg(test)]
|
||||
pub(crate) type Endpoint = crate::storage::Endpoint;
|
||||
#[cfg(test)]
|
||||
pub(crate) type Endpoints = crate::storage::Endpoints;
|
||||
#[cfg(test)]
|
||||
pub(crate) type HardLimitReader<R> = crate::storage::HardLimitReader<R>;
|
||||
#[cfg(test)]
|
||||
pub(crate) type PoolEndpoints = crate::storage::PoolEndpoints;
|
||||
#[cfg(test)]
|
||||
pub(crate) type TierConfig = ecstore_tier::tier_config::TierConfig;
|
||||
@@ -564,10 +548,6 @@ pub(crate) fn get_total_usable_capacity_free(disks: &[rustfs_madmin::Disk], info
|
||||
ecstore_capacity::get_total_usable_capacity_free(disks, info)
|
||||
}
|
||||
|
||||
pub(crate) fn is_disk_compressible(headers: &http::HeaderMap, object_name: &str) -> bool {
|
||||
ecstore_compression::is_disk_compressible(headers, object_name)
|
||||
}
|
||||
|
||||
pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) {
|
||||
ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await;
|
||||
}
|
||||
@@ -680,33 +660,6 @@ pub(crate) async fn collect_scanner_metrics_report() -> ScannerMetricsReport {
|
||||
rustfs_common::metrics::global_metrics().report().await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn boxed_reader<R>(reader: R) -> DynReader
|
||||
where
|
||||
R: crate::storage::ecstore_rio::Reader + 'static,
|
||||
{
|
||||
crate::storage::boxed_reader(reader)
|
||||
}
|
||||
|
||||
pub(crate) fn compression_metadata_value(algorithm: rustfs_utils::CompressionAlgorithm) -> String {
|
||||
crate::storage::compression_metadata_value(algorithm)
|
||||
}
|
||||
|
||||
pub(crate) fn wrap_reader<R>(reader: R) -> DynReader
|
||||
where
|
||||
R: crate::storage::ecstore_rio::ReadStream + 'static,
|
||||
{
|
||||
crate::storage::wrap_reader(reader)
|
||||
}
|
||||
|
||||
pub(crate) fn get_lock_acquire_timeout() -> tokio::time::Duration {
|
||||
crate::storage::get_lock_acquire_timeout()
|
||||
}
|
||||
|
||||
pub(crate) fn is_valid_storage_class(storage_class: &str) -> bool {
|
||||
crate::storage::is_valid_storage_class(storage_class)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<(), Error> {
|
||||
crate::storage::init_local_disks(endpoint_pools).await
|
||||
|
||||
@@ -15,8 +15,6 @@
|
||||
//! Multipart application use-case contracts.
|
||||
|
||||
use super::ECStore;
|
||||
use super::is_disk_compressible;
|
||||
use super::is_valid_storage_class;
|
||||
use super::object_api_utils::to_s3s_etag;
|
||||
use super::quota::checker::QuotaChecker;
|
||||
use super::s3_api::multipart::{
|
||||
@@ -24,11 +22,16 @@ use super::s3_api::multipart::{
|
||||
parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
|
||||
};
|
||||
use super::storage_api::access::has_bypass_governance_header;
|
||||
use super::storage_api::compression::is_disk_compressible;
|
||||
use super::storage_api::helper::OperationHelper;
|
||||
#[cfg(test)]
|
||||
use super::storage_api::io::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
|
||||
use super::storage_api::io::{HashReader, WriteEncryption, WritePlan, compression_metadata_value};
|
||||
use super::storage_api::options::{
|
||||
copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts, get_content_sha256_with_query, get_opts,
|
||||
parse_copy_source_range, put_opts, validate_archive_content_encoding,
|
||||
};
|
||||
use super::storage_api::set_disk::is_valid_storage_class;
|
||||
use super::storage_api::sse::{
|
||||
DecryptionRequest, EncryptionKeyKind, EncryptionRequest, PrepareEncryptionRequest, apply_bucket_default_lock_retention,
|
||||
build_ssec_read_headers, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
|
||||
@@ -36,9 +39,6 @@ use super::storage_api::sse::{
|
||||
mark_encrypted_multipart_metadata, sse_decryption, sse_prepare_encryption,
|
||||
};
|
||||
use super::storage_api::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader};
|
||||
#[cfg(test)]
|
||||
use super::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
|
||||
use super::{HashReader, WritePlan};
|
||||
use super::{StorageError, is_err_object_not_found, is_err_version_not_found};
|
||||
use super::{
|
||||
lifecycle::{bucket_lifecycle_audit::LcEventSrc, bucket_lifecycle_ops::enqueue_transition_immediate},
|
||||
@@ -676,7 +676,7 @@ impl DefaultMultipartUsecase {
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
super::compression_metadata_value(CompressionAlgorithm::default()),
|
||||
compression_metadata_value(CompressionAlgorithm::default()),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -895,11 +895,9 @@ impl DefaultMultipartUsecase {
|
||||
.await?
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
|
||||
let ssec_write = match ssec_material.key_kind {
|
||||
EncryptionKeyKind::Object => {
|
||||
super::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
|
||||
}
|
||||
EncryptionKeyKind::Object => WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32),
|
||||
EncryptionKeyKind::Direct => {
|
||||
super::WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id)
|
||||
WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id)
|
||||
}
|
||||
};
|
||||
write_plan = write_plan.with_encryption(ssec_write);
|
||||
@@ -915,11 +913,9 @@ impl DefaultMultipartUsecase {
|
||||
.await?
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
|
||||
let managed_write = match managed_material.key_kind {
|
||||
EncryptionKeyKind::Object => {
|
||||
super::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
|
||||
}
|
||||
EncryptionKeyKind::Object => WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32),
|
||||
EncryptionKeyKind::Direct => {
|
||||
super::WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_id)
|
||||
WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_id)
|
||||
}
|
||||
};
|
||||
write_plan = write_plan.with_encryption(managed_write);
|
||||
@@ -1241,11 +1237,9 @@ impl DefaultMultipartUsecase {
|
||||
.await?
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
|
||||
let ssec_write = match ssec_material.key_kind {
|
||||
EncryptionKeyKind::Object => {
|
||||
super::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
|
||||
}
|
||||
EncryptionKeyKind::Object => WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32),
|
||||
EncryptionKeyKind::Direct => {
|
||||
super::WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id)
|
||||
WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id)
|
||||
}
|
||||
};
|
||||
write_plan = write_plan.with_encryption(ssec_write);
|
||||
@@ -1265,11 +1259,9 @@ impl DefaultMultipartUsecase {
|
||||
.await?
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
|
||||
let managed_write = match managed_material.key_kind {
|
||||
EncryptionKeyKind::Object => {
|
||||
super::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
|
||||
}
|
||||
EncryptionKeyKind::Object => WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32),
|
||||
EncryptionKeyKind::Direct => {
|
||||
super::WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_id)
|
||||
WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_id)
|
||||
}
|
||||
};
|
||||
write_plan = write_plan.with_encryption(managed_write);
|
||||
|
||||
@@ -21,6 +21,7 @@ use super::storageclass;
|
||||
use super::ECStore;
|
||||
use super::s3_api::multipart::parse_list_parts_params;
|
||||
use super::storage_api::access::{PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut};
|
||||
use super::storage_api::compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||
use super::storage_api::concurrency::{
|
||||
self, ConcurrencyManager, GetObjectGuard, PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager,
|
||||
get_put_concurrency_aware_buffer_size,
|
||||
@@ -29,11 +30,13 @@ use super::storage_api::deadlock_detector;
|
||||
use super::storage_api::ecfs::FS;
|
||||
use super::storage_api::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children};
|
||||
use super::storage_api::helper::{OperationHelper, spawn_background_with_context};
|
||||
use super::storage_api::io::{DynReader, HashReader, WritePlan, compression_metadata_value, wrap_reader};
|
||||
use super::storage_api::options::{
|
||||
copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime_with_object_name,
|
||||
filter_object_metadata, get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, put_opts,
|
||||
};
|
||||
use super::storage_api::request_context::{self, spawn_traced};
|
||||
use super::storage_api::set_disk::{get_lock_acquire_timeout, is_valid_storage_class};
|
||||
use super::storage_api::sse::{
|
||||
DecryptionRequest, EncryptionRequest, SSEType, apply_bucket_default_lock_retention, build_ssec_read_headers,
|
||||
encryption_material_to_metadata, extract_server_side_encryption_from_headers, extract_ssec_params_from_headers,
|
||||
@@ -48,10 +51,7 @@ use super::storage_api::{
|
||||
};
|
||||
use super::{AppReplicationConfigExt as _, AppVersioningConfigExt as _, predict_lifecycle_expiration, validate_restore_request};
|
||||
use super::{DiskError, is_all_buckets_not_found};
|
||||
use super::{DynReader, HashReader, WritePlan, wrap_reader};
|
||||
use super::{Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found};
|
||||
use super::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||
use super::{get_lock_acquire_timeout, is_valid_storage_class};
|
||||
use super::{
|
||||
lifecycle::{
|
||||
bucket_lifecycle_audit::LcEventSrc,
|
||||
@@ -2383,7 +2383,7 @@ impl DefaultObjectUsecase {
|
||||
StreamReader::new(body.map(|f| f.map_err(|e| std::io::Error::other(e.to_string())))),
|
||||
);
|
||||
let algorithm = CompressionAlgorithm::default();
|
||||
insert_str(&mut metadata, SUFFIX_COMPRESSION, super::compression_metadata_value(algorithm));
|
||||
insert_str(&mut metadata, SUFFIX_COMPRESSION, compression_metadata_value(algorithm));
|
||||
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
|
||||
|
||||
let mut hrd =
|
||||
@@ -2394,7 +2394,7 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
|
||||
opts.want_checksum = hrd.checksum();
|
||||
insert_str(&mut opts.user_defined, SUFFIX_COMPRESSION, super::compression_metadata_value(algorithm));
|
||||
insert_str(&mut opts.user_defined, SUFFIX_COMPRESSION, compression_metadata_value(algorithm));
|
||||
insert_str(&mut opts.user_defined, SUFFIX_ACTUAL_SIZE, size.to_string());
|
||||
|
||||
size = HashReader::SIZE_PRESERVE_LAYER;
|
||||
@@ -3465,7 +3465,7 @@ impl DefaultObjectUsecase {
|
||||
insert_str(
|
||||
&mut compress_metadata,
|
||||
SUFFIX_COMPRESSION,
|
||||
super::compression_metadata_value(CompressionAlgorithm::default()),
|
||||
compression_metadata_value(CompressionAlgorithm::default()),
|
||||
);
|
||||
insert_str(&mut compress_metadata, SUFFIX_ACTUAL_SIZE, actual_size.to_string());
|
||||
} else {
|
||||
@@ -5037,7 +5037,7 @@ impl DefaultObjectUsecase {
|
||||
.map_err(ApiError::from)?
|
||||
} else if should_compress {
|
||||
let algorithm = CompressionAlgorithm::default();
|
||||
insert_str(&mut metadata, SUFFIX_COMPRESSION, super::compression_metadata_value(algorithm));
|
||||
insert_str(&mut metadata, SUFFIX_COMPRESSION, compression_metadata_value(algorithm));
|
||||
insert_str(&mut metadata, SUFFIX_ACTUAL_SIZE, size.to_string());
|
||||
|
||||
let hrd = HashReader::from_stream(f, size, actual_size, None, None, false).map_err(ApiError::from)?;
|
||||
|
||||
@@ -27,6 +27,10 @@ pub(crate) mod concurrency {
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod compression {
|
||||
pub(crate) use crate::storage::ecstore_compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible};
|
||||
}
|
||||
|
||||
pub(crate) mod deadlock_detector {
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::deadlock_detector::RequestHangDetectionPolicy;
|
||||
@@ -45,6 +49,12 @@ pub(crate) mod helper {
|
||||
pub(crate) use crate::storage::helper::{OperationHelper, spawn_background_with_context};
|
||||
}
|
||||
|
||||
pub(crate) mod io {
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader};
|
||||
pub(crate) use crate::storage::{DynReader, HashReader, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader};
|
||||
}
|
||||
|
||||
pub(crate) mod options {
|
||||
pub(crate) use crate::storage::options::{
|
||||
copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime,
|
||||
@@ -70,6 +80,10 @@ pub(crate) mod sse {
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod set_disk {
|
||||
pub(crate) use crate::storage::{get_lock_acquire_timeout, is_valid_storage_class};
|
||||
}
|
||||
|
||||
pub(crate) mod timeout_wrapper {
|
||||
pub(crate) use crate::storage::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper};
|
||||
}
|
||||
|
||||
@@ -163,6 +163,7 @@ RUSTFS_APP_USECASE_STORAGE_WILDCARD_HITS_FILE="${TMP_DIR}/rustfs_app_usecase_sto
|
||||
RUSTFS_APP_WILDCARD_IMPORT_HITS_FILE="${TMP_DIR}/rustfs_app_wildcard_import_hits.txt"
|
||||
RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_usecase_s3_api_bypass_hits.txt"
|
||||
RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_usecase_storage_api_bypass_hits.txt"
|
||||
RUSTFS_APP_ADMIN_STORAGE_API_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_admin_storage_api_bypass_hits.txt"
|
||||
RUSTFS_STORAGE_DIRECT_APP_CONTEXT_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_storage_direct_app_context_bypass_hits.txt"
|
||||
|
||||
awk '
|
||||
@@ -1441,15 +1442,36 @@ fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename \
|
||||
'(use crate::storage::(access|helper|options|request_context|sse|timeout_wrapper|head_prefix|concurrency|ecfs)|crate::storage::sse::EncryptionKeyKind|use crate::storage::\{|use crate::storage::[A-Z])' \
|
||||
rustfs/src/app/select_object.rs rustfs/src/app/*_usecase.rs || true
|
||||
{
|
||||
rg -n --with-filename \
|
||||
'(use crate::storage::(access|helper|options|request_context|sse|timeout_wrapper|head_prefix|concurrency|ecfs)|crate::storage::sse::EncryptionKeyKind|use crate::storage::\{|use crate::storage::[A-Z])' \
|
||||
rustfs/src/app/select_object.rs rustfs/src/app/*_usecase.rs || true
|
||||
rg -n --with-filename \
|
||||
'use super::(?:\{[^}]*\b(?:DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class)\b|(?:DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class)\b)' \
|
||||
rustfs/src/app/object_usecase.rs rustfs/src/app/multipart_usecase.rs || true
|
||||
}
|
||||
) >"$RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE" ]]; then
|
||||
report_failure "RustFS app usecases must consume storage helper APIs through rustfs/src/app/storage_api.rs: $(paste -sd '; ' "$RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename \
|
||||
'(use crate::storage::(access|request_context|ecfs)|use crate::storage::\{[^}]*Storage(ObjectInfo|ObjectOptions|PutObjReader)|crate::storage::StorageObjectOptions)' \
|
||||
rustfs/src/app/*_test.rs \
|
||||
rustfs/src/admin/router.rs \
|
||||
rustfs/src/admin/console.rs \
|
||||
rustfs/src/admin/handlers/heal.rs \
|
||||
rustfs/src/admin/handlers/metrics.rs \
|
||||
rustfs/src/admin/handlers/object_zip_download.rs || true
|
||||
) >"$RUSTFS_APP_ADMIN_STORAGE_API_BYPASS_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_APP_ADMIN_STORAGE_API_BYPASS_HITS_FILE" ]]; then
|
||||
report_failure "RustFS app/admin completed storage helper consumers must use local storage_api boundaries: $(paste -sd '; ' "$RUSTFS_APP_ADMIN_STORAGE_API_BYPASS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::app::context::|use crate::app::context|app::context::' rustfs/src/storage --glob '*.rs' |
|
||||
|
||||
@@ -21,7 +21,6 @@ cycle|infra<->interface
|
||||
dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_bucket_meta_hook
|
||||
dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_delete_bucket_hook
|
||||
dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_make_bucket_hook
|
||||
dep|rustfs/src/app/lifecycle_transition_api_test.rs|app->interface|crate::storage::ecfs::FS
|
||||
dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::ecfs::FS
|
||||
dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::bucket::ListObjectVersionsParams
|
||||
dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::bucket::ListObjectsV2Params
|
||||
|
||||
Reference in New Issue
Block a user