mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-27 16:48:58 +00:00
refactor: route app storage helper imports (#3856)
This commit is contained in:
Zhengchao An
@@ -5,9 +5,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
## Current Context
|
||||
|
||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||
- Branch: `overtrue/arch-app-s3-api-boundary`
|
||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210`.
|
||||
- Based on: latest main after PR #3853 merged.
|
||||
- Branch: `overtrue/arch-app-storage-boundary-imports`
|
||||
- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211`.
|
||||
- Based on: stacked on API-210 after PR #3853 merged.
|
||||
- PR type for this branch: `consumer-migration`
|
||||
- Runtime behavior changes: none.
|
||||
- Rust code changes: route replication pool, outbound TLS generation, runtime
|
||||
@@ -45,7 +45,7 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
readiness-time, observability, metrics, buffer, and TLS runtime source
|
||||
helpers, plus server readiness/audit/event/module-switch runtime source
|
||||
helpers, storage request/RPC/SSE runtime source helpers, and admin
|
||||
handler/service/router runtime source helpers, plus root auth/init/config/protocol/workload, app usecase, storage node-service, remaining admin grouped context import runtime source helpers, app bucket/object/multipart usecase explicit storage imports, app select/bucket/object/multipart explicit S3 DTO plus ECFS owner imports, and app-local S3 API response helper imports for bucket/object/multipart usecases,
|
||||
handler/service/router runtime source helpers, plus root auth/init/config/protocol/workload, app usecase, storage node-service, remaining admin grouped context import runtime source helpers, app bucket/object/multipart usecase explicit storage imports, app select/bucket/object/multipart explicit S3 DTO plus ECFS owner imports, app-local S3 API response helper imports for bucket/object/multipart usecases, and app-local storage helper boundary imports for select/bucket/object/multipart usecases,
|
||||
through AppContext-first or owner-crate resolver boundaries.
|
||||
- CI/script changes: lock completed owner and test/fuzz boundaries against
|
||||
bare/glob imports, scattered raw ECStore facade subpaths, and startup
|
||||
@@ -56,8 +56,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
event-bridge thin module regressions, plus IAM runtime-source bypasses;
|
||||
accept the reviewed AppContext resolver reverse dependencies in the layer
|
||||
baseline, and block direct admin AppContext resolver consumers outside the
|
||||
admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, and reject direct storage S3 API helper imports from app usecase files.
|
||||
- Docs changes: record the API-136 through API-210 owner facade and lifecycle
|
||||
admin runtime-source boundary, block root, app usecase, and storage direct AppContext resolver consumers outside their runtime-source boundaries, catch grouped AppContext imports, reject app usecase storage wildcard imports, reject app-layer S3 DTO and ECFS wildcard imports, narrow the object-usecase ECFS layer baseline entry to `FS`, reject direct storage S3 API helper imports from app usecase files, and reject direct storage helper imports from app select/usecase files.
|
||||
- Docs changes: record the API-136 through API-211 owner facade and lifecycle
|
||||
runtime-source cleanup.
|
||||
|
||||
## Phase 0 Tasks
|
||||
@@ -5025,6 +5025,22 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
layer guards, diff hygiene, residual direct S3 API scan, Rust risk scan,
|
||||
fast PR gate, and full PR gate before PR.
|
||||
|
||||
- [x] `API-211` Route app storage helper imports through app boundary.
|
||||
- Do: add an app-local storage helper boundary and route select, bucket,
|
||||
object, and multipart usecases through it instead of direct storage helper
|
||||
imports for access, helper, options, SSE, request-context, concurrency,
|
||||
prefix, timeout, and storage object helper types.
|
||||
- Acceptance: app select/usecase files no longer import the migrated storage
|
||||
helper APIs directly, layer baseline records the reviewed app boundary, and
|
||||
migration rules reject direct storage helper bypasses.
|
||||
- Must preserve: select-object preflight and traced execution, bucket request
|
||||
authorization/background metadata updates, multipart SSE/options parsing,
|
||||
object SSE/concurrency/prefix/timeout behavior, and existing storage helper
|
||||
implementations.
|
||||
- Verification: focused RustFS app compile/tests, formatting, migration and
|
||||
layer guards, diff hygiene, residual direct storage helper scan, Rust risk
|
||||
scan, fast PR gate, and full PR gate before PR.
|
||||
|
||||
## Next PRs
|
||||
|
||||
1. `consumer-migration`: continue reducing direct global reads behind AppContext resolver boundaries.
|
||||
@@ -5033,6 +5049,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
|
||||
| Expert | Status | Notes |
|
||||
|---|---|---|
|
||||
| Quality/architecture | pass | API-211 routes app select/usecase storage helper imports through an app-local storage_api boundary and adds a direct-bypass guard. |
|
||||
| Migration preservation | pass | Select preflight, bucket authorization/background work, multipart SSE/options parsing, and object SSE/concurrency/prefix/timeout paths keep the same storage helper implementations. |
|
||||
| Testing/verification | pass | Focused RustFS app compile/tests, formatting, migration/layer guards, residual direct storage helper scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. |
|
||||
| Quality/architecture | pass | API-210 routes app usecase S3 API helper imports through an app-local boundary and adds a direct-bypass guard. |
|
||||
| Migration preservation | pass | Bucket list response mapping, multipart list/part-number parsing, object list-parts parsing, and S3 owner metadata keep the same storage helper implementations. |
|
||||
| Testing/verification | pass | Focused RustFS app compile/tests, formatting, migration/layer guards, residual direct S3 API scan, Rust risk scan, fast PR gate, and full PR gate are planned before PR. |
|
||||
|
||||
@@ -22,6 +22,13 @@ use super::s3_api::bucket::{
|
||||
build_list_objects_output, build_list_objects_v2_output, parse_list_object_versions_params, parse_list_objects_v2_params,
|
||||
rustfs_owner,
|
||||
};
|
||||
use super::storage_api::StorageObjectInfo as ObjectInfo;
|
||||
use super::storage_api::access::{ReqInfo, authorize_request, req_info_ref};
|
||||
use super::storage_api::helper::{OperationHelper, spawn_background_with_context};
|
||||
use super::storage_api::{
|
||||
get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
request_context, validate_list_object_unordered_with_delimiter,
|
||||
};
|
||||
use super::{AppObjectLockConfigExt as _, AppVersioningConfigExt as _};
|
||||
use super::{
|
||||
bucket_target_sys::BucketTargetSys,
|
||||
@@ -50,13 +57,6 @@ use crate::app::runtime_sources::{
|
||||
use crate::auth::get_condition_values_with_client_info;
|
||||
use crate::error::ApiError;
|
||||
use crate::server::RemoteAddr;
|
||||
use crate::storage::StorageObjectInfo as ObjectInfo;
|
||||
use crate::storage::access::{ReqInfo, authorize_request, req_info_ref};
|
||||
use crate::storage::helper::{OperationHelper, spawn_background_with_context};
|
||||
use crate::storage::{
|
||||
get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
request_context, validate_list_object_unordered_with_delimiter,
|
||||
};
|
||||
use futures::StreamExt;
|
||||
use http::StatusCode;
|
||||
use metrics::counter;
|
||||
|
||||
@@ -23,6 +23,7 @@ pub mod object_usecase;
|
||||
pub(crate) mod runtime_sources;
|
||||
pub(crate) mod s3_api;
|
||||
mod select_object;
|
||||
pub(crate) mod storage_api;
|
||||
|
||||
#[cfg(test)]
|
||||
mod capacity_dirty_scope_test;
|
||||
|
||||
@@ -23,6 +23,19 @@ use super::s3_api::multipart::{
|
||||
ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output,
|
||||
parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number,
|
||||
};
|
||||
use super::storage_api::access::has_bypass_governance_header;
|
||||
use super::storage_api::helper::OperationHelper;
|
||||
use super::storage_api::options::{
|
||||
copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts, get_content_sha256_with_query, get_opts,
|
||||
parse_copy_source_range, put_opts, validate_archive_content_encoding,
|
||||
};
|
||||
use super::storage_api::sse::{
|
||||
DecryptionRequest, EncryptionKeyKind, EncryptionRequest, PrepareEncryptionRequest, apply_bucket_default_lock_retention,
|
||||
build_ssec_read_headers, encryption_material_to_metadata, extract_server_side_encryption_from_headers,
|
||||
extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in, map_get_object_reader_error,
|
||||
mark_encrypted_multipart_metadata, sse_decryption, sse_prepare_encryption,
|
||||
};
|
||||
use super::storage_api::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader};
|
||||
#[cfg(test)]
|
||||
use super::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader, wrap_reader};
|
||||
use super::{HashReader, WritePlan};
|
||||
@@ -38,21 +51,6 @@ use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_e
|
||||
use crate::app::runtime_sources::{AppContext, get_global_app_context, resolve_object_store_handle_for_context};
|
||||
use crate::capacity::record_capacity_write;
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::access::has_bypass_governance_header;
|
||||
use crate::storage::helper::OperationHelper;
|
||||
use crate::storage::options::{
|
||||
copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts, get_content_sha256_with_query, get_opts,
|
||||
parse_copy_source_range, put_opts, validate_archive_content_encoding,
|
||||
};
|
||||
use crate::storage::sse::{
|
||||
build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers,
|
||||
extract_ssekms_context_from_headers, map_get_object_reader_error, mark_encrypted_multipart_metadata,
|
||||
};
|
||||
use crate::storage::{
|
||||
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, apply_bucket_default_lock_retention,
|
||||
extract_server_side_encryption_from_headers, get_buffer_size_opt_in, sse_decryption, sse_prepare_encryption,
|
||||
};
|
||||
use crate::storage::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader};
|
||||
use crate::table_catalog;
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
@@ -664,7 +662,7 @@ impl DefaultMultipartUsecase {
|
||||
let ssekms_key_id = material.kms_key_id.clone();
|
||||
|
||||
let mut encryption_metadata = encryption_material_to_metadata(&material)?;
|
||||
if material.key_kind == crate::storage::sse::EncryptionKeyKind::Object {
|
||||
if material.key_kind == EncryptionKeyKind::Object {
|
||||
mark_encrypted_multipart_metadata(&mut encryption_metadata);
|
||||
}
|
||||
metadata.extend(encryption_metadata);
|
||||
@@ -896,10 +894,10 @@ impl DefaultMultipartUsecase {
|
||||
.await?
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
|
||||
let ssec_write = match ssec_material.key_kind {
|
||||
crate::storage::sse::EncryptionKeyKind::Object => {
|
||||
EncryptionKeyKind::Object => {
|
||||
super::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
|
||||
}
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => {
|
||||
EncryptionKeyKind::Direct => {
|
||||
super::WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id)
|
||||
}
|
||||
};
|
||||
@@ -916,10 +914,10 @@ impl DefaultMultipartUsecase {
|
||||
.await?
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
|
||||
let managed_write = match managed_material.key_kind {
|
||||
crate::storage::sse::EncryptionKeyKind::Object => {
|
||||
EncryptionKeyKind::Object => {
|
||||
super::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
|
||||
}
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => {
|
||||
EncryptionKeyKind::Direct => {
|
||||
super::WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_id)
|
||||
}
|
||||
};
|
||||
@@ -1235,10 +1233,10 @@ impl DefaultMultipartUsecase {
|
||||
.await?
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
|
||||
let ssec_write = match ssec_material.key_kind {
|
||||
crate::storage::sse::EncryptionKeyKind::Object => {
|
||||
EncryptionKeyKind::Object => {
|
||||
super::WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32)
|
||||
}
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => {
|
||||
EncryptionKeyKind::Direct => {
|
||||
super::WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id)
|
||||
}
|
||||
};
|
||||
@@ -1259,10 +1257,10 @@ impl DefaultMultipartUsecase {
|
||||
.await?
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
|
||||
let managed_write = match managed_material.key_kind {
|
||||
crate::storage::sse::EncryptionKeyKind::Object => {
|
||||
EncryptionKeyKind::Object => {
|
||||
super::WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32)
|
||||
}
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => {
|
||||
EncryptionKeyKind::Direct => {
|
||||
super::WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_id)
|
||||
}
|
||||
};
|
||||
@@ -1502,12 +1500,12 @@ mod tests {
|
||||
let mut encrypted_one = Vec::new();
|
||||
#[cfg(feature = "rio-v2")]
|
||||
let mut part_one_reader = match part_one_material.key_kind {
|
||||
crate::storage::sse::EncryptionKeyKind::Object => EncryptReader::new_multipart_with_object_key(
|
||||
EncryptionKeyKind::Object => EncryptReader::new_multipart_with_object_key(
|
||||
Cursor::new(part_one_plaintext.clone()),
|
||||
part_one_material.key_bytes,
|
||||
1,
|
||||
),
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => EncryptReader::new_multipart(
|
||||
EncryptionKeyKind::Direct => EncryptReader::new_multipart(
|
||||
Cursor::new(part_one_plaintext.clone()),
|
||||
part_one_material.key_bytes,
|
||||
part_one_material.base_nonce,
|
||||
@@ -1539,12 +1537,12 @@ mod tests {
|
||||
let mut encrypted_two = Vec::new();
|
||||
#[cfg(feature = "rio-v2")]
|
||||
let mut part_two_reader = match part_two_material.key_kind {
|
||||
crate::storage::sse::EncryptionKeyKind::Object => EncryptReader::new_multipart_with_object_key(
|
||||
EncryptionKeyKind::Object => EncryptReader::new_multipart_with_object_key(
|
||||
Cursor::new(part_two_plaintext.clone()),
|
||||
part_two_material.key_bytes,
|
||||
2,
|
||||
),
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => EncryptReader::new_multipart(
|
||||
EncryptionKeyKind::Direct => EncryptReader::new_multipart(
|
||||
Cursor::new(part_two_plaintext.clone()),
|
||||
part_two_material.key_bytes,
|
||||
part_two_material.base_nonce,
|
||||
@@ -1563,7 +1561,7 @@ mod tests {
|
||||
.await
|
||||
.expect("read encrypted part two");
|
||||
|
||||
if session_material.key_kind == crate::storage::sse::EncryptionKeyKind::Object {
|
||||
if session_material.key_kind == EncryptionKeyKind::Object {
|
||||
assert!(session_metadata.contains_key("X-Minio-Internal-Encrypted-Multipart"));
|
||||
assert!(session_metadata.contains_key("X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"));
|
||||
} else {
|
||||
@@ -1603,12 +1601,12 @@ mod tests {
|
||||
let plaintext_size = multipart_plaintext_size(&parts, -1);
|
||||
#[cfg(feature = "rio-v2")]
|
||||
let decrypted_stream = match decryption_material.key_kind {
|
||||
crate::storage::sse::EncryptionKeyKind::Object => boxed_reader(DecryptReader::new_multipart_with_object_key(
|
||||
EncryptionKeyKind::Object => boxed_reader(DecryptReader::new_multipart_with_object_key(
|
||||
wrap_reader(Cursor::new(encrypted_stream)),
|
||||
decryption_material.key_bytes,
|
||||
multipart_part_numbers(&parts),
|
||||
)),
|
||||
crate::storage::sse::EncryptionKeyKind::Direct => boxed_reader(DecryptReader::new_multipart(
|
||||
EncryptionKeyKind::Direct => boxed_reader(DecryptReader::new_multipart(
|
||||
wrap_reader(Cursor::new(encrypted_stream)),
|
||||
decryption_material.key_bytes,
|
||||
decryption_material.base_nonce,
|
||||
|
||||
@@ -20,6 +20,32 @@ use super::storageclass;
|
||||
// Performance metrics recording (with zero-copy-metrics integration)
|
||||
use super::ECStore;
|
||||
use super::s3_api::multipart::parse_list_parts_params;
|
||||
use super::storage_api::access::{PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut};
|
||||
use super::storage_api::concurrency::{
|
||||
self, ConcurrencyManager, GetObjectGuard, PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager,
|
||||
get_put_concurrency_aware_buffer_size,
|
||||
};
|
||||
use super::storage_api::deadlock_detector;
|
||||
use super::storage_api::ecfs::FS;
|
||||
use super::storage_api::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children};
|
||||
use super::storage_api::helper::{OperationHelper, spawn_background_with_context};
|
||||
use super::storage_api::options::{
|
||||
copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime_with_object_name,
|
||||
filter_object_metadata, get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, put_opts,
|
||||
};
|
||||
use super::storage_api::request_context::{self, spawn_traced};
|
||||
use super::storage_api::sse::{
|
||||
DecryptionRequest, EncryptionRequest, SSEType, apply_bucket_default_lock_retention, build_ssec_read_headers,
|
||||
encryption_material_to_metadata, extract_server_side_encryption_from_headers, extract_ssec_params_from_headers,
|
||||
extract_ssekms_context_from_headers, get_buffer_size_opt_in, map_get_object_reader_error, sse_decryption, sse_encryption,
|
||||
};
|
||||
use super::storage_api::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper};
|
||||
use super::storage_api::{
|
||||
RFC1123, check_preconditions, get_validated_store, has_replication_rules, parse_object_lock_legal_hold,
|
||||
parse_object_lock_retention, parse_part_number_i32_to_usize, remove_object_lock_metadata_for_copy,
|
||||
strip_managed_encryption_metadata, validate_bucket_object_lock_enabled, validate_object_key, validate_sse_headers_for_read,
|
||||
validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors,
|
||||
};
|
||||
use super::{AppReplicationConfigExt as _, AppVersioningConfigExt as _, predict_lifecycle_expiration, validate_restore_request};
|
||||
use super::{DiskError, is_all_buckets_not_found};
|
||||
use super::{DynReader, HashReader, WritePlan, wrap_reader};
|
||||
@@ -53,32 +79,6 @@ use crate::config::RustFSBufferConfig;
|
||||
use crate::delete_tail_activity::{DeleteTailActivityGuard, DeleteTailStage};
|
||||
use crate::error::ApiError;
|
||||
use crate::server::convert_ecstore_object_info;
|
||||
use crate::storage::access::{PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut};
|
||||
use crate::storage::concurrency::{
|
||||
ConcurrencyManager, GetObjectGuard, PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager,
|
||||
get_put_concurrency_aware_buffer_size,
|
||||
};
|
||||
use crate::storage::ecfs::FS;
|
||||
use crate::storage::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children};
|
||||
use crate::storage::helper::{OperationHelper, spawn_background_with_context};
|
||||
use crate::storage::options::{
|
||||
copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime_with_object_name,
|
||||
filter_object_metadata, get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, put_opts,
|
||||
};
|
||||
use crate::storage::request_context::spawn_traced;
|
||||
use crate::storage::sse::{
|
||||
SSEType, build_ssec_read_headers, encryption_material_to_metadata, extract_ssekms_context_from_headers,
|
||||
map_get_object_reader_error,
|
||||
};
|
||||
use crate::storage::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper};
|
||||
use crate::storage::{
|
||||
DecryptionRequest, EncryptionRequest, RFC1123, apply_bucket_default_lock_retention, check_preconditions, concurrency,
|
||||
deadlock_detector, extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, get_buffer_size_opt_in,
|
||||
get_validated_store, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention,
|
||||
parse_part_number_i32_to_usize, remove_object_lock_metadata_for_copy, request_context, sse_decryption, sse_encryption,
|
||||
strip_managed_encryption_metadata, validate_bucket_object_lock_enabled, validate_object_key, validate_sse_headers_for_read,
|
||||
validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors,
|
||||
};
|
||||
use crate::table_catalog;
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
@@ -155,7 +155,7 @@ use tokio_util::io::{ReaderStream, StreamReader};
|
||||
use tracing::{debug, error, instrument, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::storage::{
|
||||
use super::storage_api::{
|
||||
StorageDeletedObject, StorageObjectInfo as ObjectInfo, StorageObjectOptions as ObjectOptions,
|
||||
StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader,
|
||||
};
|
||||
@@ -638,7 +638,7 @@ fn should_buffer_get_object_in_memory_with_threshold(
|
||||
#[cfg(test)]
|
||||
mod deadlock_request_guard_tests {
|
||||
use super::DeadlockRequestGuard;
|
||||
use crate::storage::deadlock_detector::{DeadlockDetector, RequestHangDetectionPolicy};
|
||||
use crate::app::storage_api::deadlock_detector::{DeadlockDetector, RequestHangDetectionPolicy};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use super::storage_api::options::get_opts;
|
||||
use super::storage_api::request_context::spawn_traced;
|
||||
use super::storage_api::{get_validated_store, validate_sse_headers_for_read, validate_ssec_for_read};
|
||||
use crate::app::runtime_sources::resolve_s3select_db;
|
||||
use crate::error::ApiError;
|
||||
use crate::storage::options::get_opts;
|
||||
use crate::storage::request_context::spawn_traced;
|
||||
use crate::storage::{get_validated_store, validate_sse_headers_for_read, validate_ssec_for_read};
|
||||
use bytes::Bytes;
|
||||
use datafusion::arrow::{
|
||||
csv::{QuoteStyle, WriterBuilder as CsvWriterBuilder, writer::Terminator},
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! App-local boundary for storage-layer helper APIs used by S3 use cases.
|
||||
|
||||
pub(crate) mod access {
|
||||
pub(crate) use crate::storage::access::{
|
||||
PostObjectRequestMarker, ReqInfo, authorize_request, has_bypass_governance_header, req_info_mut, req_info_ref,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod concurrency {
|
||||
pub(crate) use crate::storage::concurrency::{
|
||||
ConcurrencyManager, GetObjectGuard, IoQueueStatus, IoStrategy, PutObjectGuard, get_concurrency_aware_buffer_size,
|
||||
get_concurrency_manager, get_put_concurrency_aware_buffer_size,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod deadlock_detector {
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::deadlock_detector::RequestHangDetectionPolicy;
|
||||
pub(crate) use crate::storage::deadlock_detector::{DeadlockDetector, get_deadlock_detector};
|
||||
}
|
||||
|
||||
pub(crate) mod ecfs {
|
||||
pub(crate) use crate::storage::ecfs::FS;
|
||||
}
|
||||
|
||||
pub(crate) mod head_prefix {
|
||||
pub(crate) use crate::storage::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children};
|
||||
}
|
||||
|
||||
pub(crate) mod helper {
|
||||
pub(crate) use crate::storage::helper::{OperationHelper, spawn_background_with_context};
|
||||
}
|
||||
|
||||
pub(crate) mod options {
|
||||
pub(crate) use crate::storage::options::{
|
||||
copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime,
|
||||
extract_metadata_from_mime_with_object_name, filter_object_metadata, get_complete_multipart_upload_opts,
|
||||
get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, parse_copy_source_range, put_opts,
|
||||
validate_archive_content_encoding,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod request_context {
|
||||
pub(crate) use crate::storage::request_context::{RequestContext, spawn_traced};
|
||||
}
|
||||
|
||||
pub(crate) mod sse {
|
||||
pub(crate) use crate::storage::sse::{
|
||||
EncryptionKeyKind, SSEType, build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers,
|
||||
extract_ssekms_context_from_headers, map_get_object_reader_error, mark_encrypted_multipart_metadata,
|
||||
};
|
||||
pub(crate) use crate::storage::{
|
||||
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, apply_bucket_default_lock_retention,
|
||||
extract_server_side_encryption_from_headers, get_buffer_size_opt_in, sse_decryption, sse_encryption,
|
||||
sse_prepare_encryption,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod timeout_wrapper {
|
||||
pub(crate) use crate::storage::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper};
|
||||
}
|
||||
|
||||
pub(crate) use crate::storage::{
|
||||
RFC1123, StorageDeletedObject, StorageObjectInfo, StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader,
|
||||
check_preconditions, get_validated_store, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention,
|
||||
parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata, validate_bucket_object_lock_enabled,
|
||||
validate_list_object_unordered_with_delimiter, validate_object_key, validate_sse_headers_for_read,
|
||||
validate_sse_headers_for_write, validate_ssec_for_read, wrap_response_with_cors,
|
||||
};
|
||||
@@ -50,9 +50,9 @@ mod sse_test;
|
||||
|
||||
pub(crate) use ecfs_extend::*;
|
||||
pub(crate) use sse::{
|
||||
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, extract_server_side_encryption_from_headers,
|
||||
extract_ssec_params_from_headers, sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata,
|
||||
validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read,
|
||||
DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, extract_server_side_encryption_from_headers, sse_decryption,
|
||||
sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read,
|
||||
validate_sse_headers_for_write, validate_ssec_for_read,
|
||||
};
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -162,6 +162,7 @@ RUSTFS_APP_USECASE_RUNTIME_SOURCE_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_usecas
|
||||
RUSTFS_APP_USECASE_STORAGE_WILDCARD_HITS_FILE="${TMP_DIR}/rustfs_app_usecase_storage_wildcard_hits.txt"
|
||||
RUSTFS_APP_WILDCARD_IMPORT_HITS_FILE="${TMP_DIR}/rustfs_app_wildcard_import_hits.txt"
|
||||
RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_usecase_s3_api_bypass_hits.txt"
|
||||
RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_usecase_storage_api_bypass_hits.txt"
|
||||
RUSTFS_STORAGE_DIRECT_APP_CONTEXT_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_storage_direct_app_context_bypass_hits.txt"
|
||||
|
||||
awk '
|
||||
@@ -1438,6 +1439,17 @@ if [[ -s "$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE" ]]; then
|
||||
report_failure "RustFS app usecases must consume S3 API helpers through rustfs/src/app/s3_api.rs: $(paste -sd '; ' "$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename \
|
||||
'(use crate::storage::(access|helper|options|request_context|sse|timeout_wrapper|head_prefix|concurrency|ecfs)|crate::storage::sse::EncryptionKeyKind|use crate::storage::\{|use crate::storage::[A-Z])' \
|
||||
rustfs/src/app/select_object.rs rustfs/src/app/*_usecase.rs || true
|
||||
) >"$RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE" ]]; then
|
||||
report_failure "RustFS app usecases must consume storage helper APIs through rustfs/src/app/storage_api.rs: $(paste -sd '; ' "$RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE")"
|
||||
fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::app::context::|use crate::app::context|app::context::' rustfs/src/storage --glob '*.rs' |
|
||||
|
||||
@@ -22,7 +22,7 @@ dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site
|
||||
dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_delete_bucket_hook
|
||||
dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_make_bucket_hook
|
||||
dep|rustfs/src/app/lifecycle_transition_api_test.rs|app->interface|crate::storage::ecfs::FS
|
||||
dep|rustfs/src/app/object_usecase.rs|app->interface|crate::storage::ecfs::FS
|
||||
dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::ecfs::FS
|
||||
dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::bucket::ListObjectVersionsParams
|
||||
dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::bucket::ListObjectsV2Params
|
||||
dep|rustfs/src/app/s3_api.rs|app->interface|crate::storage::s3_api::bucket::build_list_buckets_output
|
||||
|
||||
Reference in New Issue
Block a user