diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index bb945d35d..60523c9cc 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,94 +5,22 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-admin-contract-domain-batch` -- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221/API-222/API-223/API-224/API-225/API-226/API-227/API-228/API-229/API-230/API-231/API-232/API-233/API-234/API-235/API-236/API-237/API-238/API-239/API-240/API-241/API-242/API-243/API-244/API-245/CTX-002`. -- Based on: rebased onto current `origin/main` after PR #3911 merged. +- Branch: `overtrue/arch-storage-owner-rpc-domain-batch` +- Baseline: completed `C-011/C-012/C-013/API-055/API-059/API-079/API-080/API-081/API-082/API-083/API-084/API-085/API-086/API-087/API-088/API-089/API-090/API-091/API-092/API-093/API-094/API-095/API-096/API-097/API-098/API-099/API-100/API-101/API-102/API-103/API-104/API-105/API-106/API-107/API-108/API-109/API-110/API-111/API-112/API-113/API-114/API-115/API-116/API-117/API-118/API-119/API-120/API-121/API-122/API-123/API-124/API-125/API-126/API-127/API-128/API-129/API-130/API-131/API-132/API-133/API-134/API-135/API-136/API-137/API-138/API-139/API-140/API-141/API-142/API-143/API-144/API-145/API-146/API-147/API-148/API-149/API-150/API-151/API-152/API-153/API-154/API-155/API-156/API-157/API-158/API-159/API-160/API-161/API-162/API-163/API-164/API-165/API-166/API-167/API-168/API-169/API-170/API-171/API-172/API-173/API-174/API-175/API-176/API-177/API-178/API-179/API-180/API-181/API-182/API-183/API-184/API-185/API-186/API-187/API-188/API-189/API-190/API-191/API-192/API-193/API-194/API-195/API-196/API-197/API-198/API-199/API-200/API-201/API-202/API-203/API-204/API-205/API-206/API-207/API-208/API-209/API-210/API-211/API-212/API-213/API-214/API-215/API-216/API-217/API-218/API-219/API-220/API-221/API-222/API-223/API-224/API-225/API-226/API-227/API-228/API-229/API-230/API-231/API-232/API-233/API-234/API-235/API-236/API-237/API-238/API-239/API-240/API-241/API-242/API-243/API-244/API-245/API-246/API-247/API-248/API-249/API-250/API-251/API-252/API-253/API-254/CTX-002`. +- Current phase PR: API-247 through API-255 storage owner/domain boundary + cleanup. +- Based on: rebased onto current `origin/main` after prerequisite PRs #3911, + #3913, and #3914 merged. - PR type for this branch: `consumer-migration` -- Runtime behavior changes: none expected for API-246; admin code still uses - the same storage contracts, now exposed from admin `contract` domain modules - instead of its flat root facade; rebase-exposed app usecase compile fixes - keep the same stream and data-usage behavior. -- Rust code changes: segment the admin local `contract` facade into domain - modules, migrate admin consumers to domain contract imports, reject flat - admin contract consumers in migration guardrails, route - replication pool, outbound TLS generation, runtime region, KMS encryption - service, runtime support handles, S3 Select DB, - internode RPC metrics, IAM authorization/handler reads, notification - rules/event dispatch, admin OIDC/token-signing reads, IAM root credential - consumers, IAM OIDC config reads, scanner runtime-config reads, OBS metrics - runtime source reads, RIO HTTP reader TLS/metrics runtime source reads, and - gRPC/transition network client TLS/metrics runtime source reads, plus ECStore - data-plane KMS/storage-class/deployment-id/lock-manager/erasure metric reads, - plus ECStore observability/status object-store, endpoint, node-name, - boot-time, init-time, root-disk threshold, and cached RPC channel reads, - plus ECStore replication pool, replication stats, and event-host reads, - plus ECStore lifecycle queue state, tier config, lifecycle config, deployment - id, event-host reads, bucket monitor reads, replication worker pool reads, - config/tier first-node checks, rebalance endpoint-locality checks, bucket - metadata object-store reads, metadata endpoint/setup reads, lifecycle - object-store reads, replication object-store readiness checks, and ECStore - pools/tier/notification owner-root runtime source reads, plus ECStore setup - state, boot-time, endpoint snapshot, local node, local disk map, and lock - client reads in store initialization, sets, set-disk, pool-space, and peer - client paths, plus ECStore facade server-config, storage-class, - notification, bucket-metadata, endpoint, region, tier-config, server-address, - object-store publication, lock-client publication, and local-node publication - paths, plus ECStore batch processor, dynamic storage-class publication, - RustFS cluster snapshot facade alias paths, ECStore RPC test runtime global - helpers, ECStore lifecycle queue and transition state handle facades, RustFS - AppContext lifecycle expiry-state resolver paths, RustFS app/admin test - runtime-source helpers, scanner lifecycle/tier runtime source reads, and the - stale RustFS tier-config and expiry-state test compat shims, plus scanner - runtime config, erasure-mode, lifecycle queue, and tier runtime source helper - names, plus IAM root-credential, server-config, and notification runtime - source helpers, plus AppContext fallback KMS/IAM/ECStore/config/metrics/TLS - runtime source helpers, plus startup/root KMS, credentials, region, - readiness-time, observability, metrics, buffer, and TLS runtime source - helpers, plus server readiness/audit/event/module-switch runtime source - helpers, storage request/RPC/SSE runtime source helpers, and admin - handler/service/router runtime source helpers, plus root auth/init/config/protocol/workload, app usecase, storage node-service, remaining admin grouped context import runtime source helpers, app bucket/object/multipart usecase explicit storage imports, app select/bucket/object/multipart explicit S3 DTO plus ECFS owner imports, app-local S3 API response helper imports for bucket/object/multipart usecases, app-local storage helper boundary imports for select/bucket/object/multipart usecases, admin/app test storage helper boundary imports for request context, authorization, object options, and ECFS test harness consumers, app storage IO/compression/set-disk helper boundary imports for object and multipart usecases, app storage error, ETag, and storage-class helper boundary imports for bucket/object/multipart usecases and lifecycle transition tests, app bucket owner facade imports for lifecycle, metadata, object-lock, quota, replication, tagging, target, versioning, and transition test helpers, and app/admin runtime, capacity, data-usage, endpoint, and global facade imports, - admin root storage facade consumers for config, bucket metadata, replication, - rebalancing, tier, quota, metrics, object zip, site replication, and admin - service config paths, through AppContext-first or owner-crate resolver - boundaries, plus root/server/startup storage facade consumers for startup - storage, bucket metadata, notification, services, readiness, HTTP/layer - request context, module switches, event dispatch, cluster/runtime snapshots, - capacity, workload admission, table catalog, init, protocol clients, and - config tests through a root-local storage_api boundary, plus root/server/startup - `rustfs_storage_api` contract imports through the same boundary, app/admin - `rustfs_storage_api` contract imports through their local boundaries, app - S3 helper forwarding through `app::storage_api`, scanner/heal source and - test ECStore plus storage contract imports through crate-local `storage_api` - boundaries, and remaining IAM, notify, OBS metrics, Swift, S3 Select, e2e, - and fuzz ECStore/storage contract imports through local `storage_api` - boundaries, plus storage owner root ECStore facade and storage contract - aggregation through `rustfs/src/storage/storage_api.rs`, storage owner - submodule storage contract imports through the same owner-local boundary, - ECStore internal storage contract imports through the owner-local - `storage_api_contracts` boundary, admin system, pool, cluster snapshot, - plugin catalog, table catalog, module-switch, and console admin discovery - `DefaultAdminUsecase` construction through `admin::runtime_sources`, storage - ECFS S3 route app usecase construction through `storage::s3_api`, root - storage API consumers through domain modules for startup, server, cluster, - table, protocols, capacity, workload, config tests, and error mapping, and - admin storage API consumers through admin domain modules for access, bucket, - cluster, config, contract, error, metrics, object, rebalance, runtime, and - tier boundaries, plus app storage API consumers through app domain modules - for admin, bucket, object, multipart, select, context, and test boundaries, - plus scanner, heal, IAM, OBS metrics, S3 Select, Swift, e2e, and related test - storage API consumers through external crate-local domain modules, plus - residual notify crate-boundary consumers, ECStore test/bench storage API - domain modules, and local ECStore type aliases for admin config and storage - RPC paths, plus external/test local storage API contract imports through - local `storage_contracts` aliases and consumer-domain modules instead of - root re-exports, plus RustFS storage owner, admin, and app local storage API - contract imports through local `storage_contracts` aliases and domain-module - exports instead of root re-exports, plus RustFS app/admin storage helper - imports through domain-module exports instead of root re-exports, plus - ECStore owner-local `storage_api_contracts` consumers import bucket, list, - multipart, object, admin, topology, range, namespace, heal, lifecycle, and - error contracts through domain modules instead of the flat root facade. +- Runtime behavior changes: none expected for API-255; root, app, and admin + facade modules consume the same storage owner symbols through + `crate::storage::storage_api` instead of direct storage root/module paths. +- Rust code changes: migrate `rustfs/src/storage_api.rs`, + `rustfs/src/app/storage_api.rs`, and `rustfs/src/admin/storage_api.rs` to + owner `storage_api` consumer modules; expose the needed access, + concurrency, deadlock, request-context, SSE, timeout, S3 API, and helper + consumer modules from the storage owner boundary; remove the storage root SSE + re-export. - CI/script changes: lock completed owner and test/fuzz boundaries against bare/glob imports, scattered raw ECStore facade subpaths, and startup runtime/root-server/table/S3/app shared/app bucket/app ECStore/admin facade @@ -109,8 +37,18 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block cleanup, plus external crate-local and residual local storage API consumer-domain cleanup, external/test storage contract root re-export cleanup, RustFS local storage contract root re-export cleanup, RustFS - app/admin storage helper root re-export cleanup, and ECStore - storage_api_contracts domain segmentation. + app/admin storage helper root re-export cleanup, ECStore + storage_api_contracts domain segmentation, root storage contract facade + domain segmentation, root runtime facade consumer-domain segmentation, and + storage-owner internal consumer-domain cleanup; reject storage-owner internal + root contract/object helper facade consumers after API-250, reject + storage-owner runtime source, object-lock helper, RPC relative root, and ECFS + test root consumers after API-251, reject restored RPC wildcard imports + after API-252, reject restored parent wildcard imports anywhere under + `rustfs/src/storage` after API-253, reject restoring storage owner root + wildcard re-exports after API-254, reject direct storage owner paths from the + root/app/admin storage facades after API-255, and reject restoring storage + root SSE re-exports after API-255. ## Phase 0 Tasks @@ -5633,19 +5571,170 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block service config call paths. - Verification: focused RustFS compile, formatting, migration/layer guards, admin contract root consumer scan, diff hygiene, and Rust risk scan passed; - full PR gate passed before PR. + full PR gate is planned before PR. + +- [x] `API-247` Segment app usecase contract facades by domain module. + - Do: split app admin, bucket, object, multipart, select, and test usecase + storage contract exports into domain modules, then migrate app consumers to + those modules. + - Acceptance: app usecase consumers no longer import storage contract symbols + from usecase root facades, and migration rules reject regressions. + - Must preserve: admin info, bucket metadata/listing, object, multipart, + select-object, and app capacity dirty-scope test call paths. + - Verification: focused RustFS compile, formatting, migration/layer guards, + app usecase contract root consumer scan, diff hygiene, and Rust risk scan + passed; full PR gate is planned before PR. + +- [x] `API-248` Segment root storage contract facades by domain module. + - Do: split root cluster, error, server, startup, and table storage contract + exports into domain modules, then migrate root consumers to those modules. + - Acceptance: root consumers no longer import storage contract symbols from + root storage_api facades, and migration rules reject regressions. + - Must preserve: runtime capabilities, cluster snapshots, server readiness, + server event tests, startup bucket metadata migration, API error mapping, + and table catalog contract call paths. + - Verification: focused RustFS compile, formatting, migration/layer guards, + root storage contract root consumer scan, diff hygiene, and Rust risk scan + passed; full PR gate is planned before PR. + +- [x] `API-249` Segment root runtime storage facades by consumer domain. + - Do: split startup, server, protocols, capacity, and workload root runtime + facade exports into consumer-domain modules, then migrate root consumers. + - Acceptance: root consumers no longer import migrated runtime symbols from + root storage_api facades, and migration rules reject regressions. + - Must preserve: startup storage/services/metadata/notification paths, server + HTTP/layer/event/readiness/module-switch/runtime-source paths, protocol + storage client setup, capacity disk reporting, and workload admission. + - Verification: focused RustFS compile, formatting, migration/layer guards, + root runtime facade root consumer scan, diff hygiene, and Rust risk scan + passed; full PR gate is planned before PR. + +- [x] `API-250` Segment storage-owner internal consumers by domain. + - Do: route storage-owner internal ECFS, ECFS extension, request options, + access, helper, S3 API, and RPC node service contract/object helper + consumers through owner-local consumer-domain modules. + - Acceptance: migrated storage-owner internals no longer import completed + contract/object helper symbols from flat `crate::storage` root facades, and + migration rules reject regressions. + - Must preserve: ECFS object/bucket flows, object-lock option parsing, S3 + list/multipart DTO projections, request authorization, audit/notification + helpers, and node RPC bucket/admin calls. + - Verification: focused RustFS compile, formatting, migration/layer guards, + storage-owner root facade consumer scan, diff hygiene, and Rust risk scan + passed; full PR gate is planned before PR. + +- [x] `API-251` Segment storage-owner runtime and helper consumers by domain. + - Do: route storage-owner runtime source imports, node/HTTP RPC root aliases, + RPC relative root aliases, ECFS object-lock helpers, and ECFS test helper + consumers through owner-local consumer-domain modules. + - Acceptance: migrated storage-owner runtime/helper/test consumers no longer + import completed runtime, object-lock, or ECFS test helpers from flat root + storage facades, and migration rules reject regressions. + - Must preserve: ECFS object-lock flows, runtime source resolution, node RPC + service behavior, HTTP RPC buffer sizing, and ECFS helper/tag test coverage. + - Verification: focused RustFS compile/test compile, formatting, + migration/layer guards, storage-owner runtime/helper/test root consumer + scans, diff hygiene, and Rust risk scan passed; full PR gate is planned + before PR. + +- [x] `API-252` Make storage-owner RPC imports explicit. + - Do: replace parent wildcard imports in storage-owner RPC bucket, disk, + event, health, HTTP service, lock, metrics, and msgpack tests with explicit + owner-domain imports, then prune stale node-service aggregate imports. + - Acceptance: migrated RPC modules no longer rely on `use super::*`, and + migration rules reject regressions. + - Must preserve: node RPC handler behavior, HTTP RPC routing and streaming, + metric serialization, lock response mapping, bucket peer calls, and msgpack + compatibility tests. + - Verification: focused RustFS compile/test compile, formatting, + migration/layer guards, storage-owner RPC wildcard scan, diff hygiene, and + Rust risk scan passed; full PR gate is planned before PR. + +- [x] `API-253` Remove storage-owner test wildcard imports directory-wide. + - Do: replace every remaining `use super::*` under `rustfs/src/storage` with + explicit test imports across access, SSE, RPC, concurrency, timeout, + helper, backpressure, lock optimizer, options, request context, and + deadlock detector tests. + - Acceptance: `rustfs/src/storage` no longer contains parent wildcard + imports, and migration rules reject restored storage-wide parent wildcard + imports. + - Must preserve: storage access authorization tests, SSE/KMS tests, node RPC + tests, concurrency scheduling tests, timeout/backpressure helpers, option + parsing, request context extraction, and deadlock detector coverage. + - Verification: focused RustFS compile/test compile, formatting, + migration/layer guards, storage-wide wildcard scan, diff hygiene, and Rust + risk scan passed; full PR gate is planned before PR. + +- [x] `API-254` Make storage-owner root exports explicit. + - Do: expose the storage-owner `storage_api` module as a crate-local boundary + and replace the storage root wildcard re-export with an explicit + re-export surface. + - Acceptance: `rustfs/src/storage/mod.rs` no longer uses + `pub(crate) use storage_api::*`, and migration rules reject restoring that + wildcard root export. + - Must preserve: all existing root, app, admin, storage, RPC, S3 API, ECFS, + SSE, startup, metrics, table, and test call paths that still consume the + storage owner crate-local API surface. + - Verification: focused RustFS compile, formatting, migration/layer guards, + root export wildcard scan, diff hygiene, and Rust risk scan passed; full PR + gate is planned before PR. + +- [x] `API-255` Migrate root, app, and admin storage facades to owner storage_api. + - Do: route the RustFS root, app, and admin storage facade files through + `crate::storage::storage_api` owner consumer modules instead of direct + `crate::storage` root/module paths. + - Acceptance: `rustfs/src/storage_api.rs`, + `rustfs/src/app/storage_api.rs`, and `rustfs/src/admin/storage_api.rs` no + longer contain direct `crate::storage` owner paths outside + `crate::storage::storage_api`, and migration rules reject restoring those + bypasses. + - Must preserve: root startup/server/protocol/workload/table paths, app + bucket/object/multipart/runtime/SSE paths, and admin metrics/bucket/access + facade behavior. + - Verification: focused RustFS compile, formatting, direct facade bypass + scan, migration guard, diff hygiene, and Rust risk scan passed; full PR + gate is planned before PR. ## Next PRs -1. `consumer-migration`: continue larger owner boundary batches after API-246. +1. `consumer-migration`: continue larger same-class storage owner cleanup + batches, prioritizing remaining root re-exports and consumer-domain guard + tightening. ## Pre-Push Review Log | Expert | Status | Notes | |---|---|---| +| Quality/architecture | pass | API-255 moves the root, app, and admin storage facades onto the owner `storage_api` boundary and removes the remaining root SSE re-export. | +| Migration preservation | pass | Root startup/server/protocol/workload/table paths, app bucket/object/multipart/runtime/SSE paths, and admin metrics/bucket/access facades keep the same underlying storage symbols. | +| Testing/verification | pass | Focused RustFS compile, formatting, direct facade bypass scan, migration guard, diff hygiene, and diff-added Rust risk scan passed; full PR gate is planned before PR. | +| Quality/architecture | pass | API-254 removes the storage-owner root wildcard export and keeps the owner API surface explicit at the module boundary. | +| Migration preservation | pass | Existing root, app, admin, storage, RPC, S3 API, ECFS, SSE, startup, metrics, table, and test consumers keep the same owner symbols. | +| Testing/verification | pass | Focused RustFS compile, formatting, migration/layer guards, root export wildcard scan, diff hygiene, and diff-added Rust risk scan passed; full PR gate is planned before PR. | +| Quality/architecture | pass | API-253 removes parent wildcard imports across the full storage-owner directory test surface and promotes the guard from RPC-only to storage-wide. | +| Migration preservation | pass | Access, SSE, RPC, concurrency, timeout, helper, backpressure, lock optimizer, options, request context, and deadlock detector tests keep the same underlying symbols. | +| Testing/verification | pass | RustFS compile/test compile, formatting, migration/layer guards, storage-wide wildcard scan, diff hygiene, and diff-added Rust risk scan passed; full PR gate is planned before PR. | +| Quality/architecture | pass | API-252 replaces parent wildcard imports across the storage-owner RPC module family with explicit owner-domain imports. | +| Migration preservation | pass | Node RPC handlers, HTTP RPC routing/streaming, metric serialization, lock mapping, bucket peer calls, and msgpack tests keep the same underlying symbols. | +| Testing/verification | pass | Focused RustFS compile/test compile, formatting, migration/layer guards, storage-owner RPC wildcard scan, diff hygiene, and diff-added Rust risk scan passed; full PR gate is planned before PR. | +| Quality/architecture | pass | API-251 routes storage-owner runtime sources, RPC root/relative aliases, object-lock helpers, and ECFS test helpers through owner-local consumer-domain modules. | +| Migration preservation | pass | Runtime source resolution, node/HTTP RPC behavior, ECFS object-lock flows, and ECFS helper/tag tests keep the same underlying symbols. | +| Testing/verification | pass | Focused RustFS compile/test compile, formatting, migration/layer guards, storage-owner runtime/helper/test root consumer scans, diff hygiene, and diff-added Rust risk scan passed; full PR gate is planned before PR. | +| Quality/architecture | pass | API-250 routes storage-owner internal contract/object helper consumers through owner-local consumer-domain modules. | +| Migration preservation | pass | ECFS, S3 list/multipart projections, option parsing, access checks, helper, and node RPC paths keep the same underlying storage symbols. | +| Testing/verification | pass | Focused RustFS compile, formatting, migration/layer guards, storage-owner root facade consumer scan, diff hygiene, and diff-added Rust risk scan passed; full PR gate is planned before PR. | +| Quality/architecture | pass | API-249 segments root runtime storage facade exports into consumer-domain modules instead of root facade exposure. | +| Migration preservation | pass | Startup, server, protocols, capacity, and workload call paths keep the same underlying storage symbols and behavior. | +| Testing/verification | pass | Focused RustFS compile, formatting, migration/layer guards, root runtime facade root consumer scan, diff hygiene, and diff-added Rust risk scan passed; full PR gate is planned before PR. | +| Quality/architecture | pass | API-248 segments root storage contract exports into domain modules instead of root facade exposure. | +| Migration preservation | pass | Runtime capabilities, cluster snapshots, server readiness, server event tests, startup bucket metadata migration, API error mapping, and table catalog paths keep the same underlying contracts. | +| Testing/verification | pass | Focused RustFS compile, formatting, migration/layer guards, root storage contract root consumer scan, diff hygiene, and diff-added Rust risk scan passed; full PR gate is planned before PR. | +| Quality/architecture | pass | API-247 segments app usecase contract exports into domain modules instead of root usecase facade exposure. | +| Migration preservation | pass | Admin info, bucket metadata/listing, object, multipart, select-object, and app capacity dirty-scope test paths keep the same underlying contracts. | +| Testing/verification | pass | Focused RustFS compile, formatting, migration/layer guards, app usecase contract root consumer scan, diff hygiene, and diff-added Rust risk scan passed; full PR gate is planned before PR. | | Quality/architecture | pass | API-246 segments the admin local contract facade into domain modules instead of flat root contract exposure. | | Migration preservation | pass | Bucket metadata, replication, site replication, rebalance, account info, config admin, heal, object zip download, router, and admin service config paths keep the same underlying contracts. | -| Testing/verification | pass | Focused RustFS compile, formatting, migration/layer guards, admin contract root consumer scan, diff hygiene, diff-added Rust risk scan, and full PR gate passed before PR. | +| Testing/verification | pass | Focused RustFS compile, formatting, migration/layer guards, admin contract root consumer scan, diff hygiene, and diff-added Rust risk scan passed; full PR gate is planned before PR. | | Quality/architecture | pass | API-245 segments the storage-owner local contract facade into domain modules instead of flat root contract exposure. | | Migration preservation | pass | ECFS bucket/object calls, storage RPC bucket/admin calls, S3 list/multipart DTO projections, request option parsing, access checks, and topology snapshot construction keep the same underlying contracts. | | Testing/verification | pass | Focused RustFS compile, formatting, migration/layer guards, storage-owner raw contract/root consumer scans, diff hygiene, and diff-added Rust risk scan passed; full PR gate is planned before PR. | @@ -5922,9 +6011,149 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block Passed before push: +- Issue #660 API-255 current slice: + - Branch freshness check: stacked on + `overtrue/arch-storage-owner-root-export-cleanup` while prerequisite PRs + are pending. + - `cargo check -p rustfs --lib`: passed. + - `cargo test -p rustfs --lib storage --no-run`: passed with existing + `StorageObjectInfo` dead code warning. + - `cargo fmt --all`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed after removing stale app + storage S3 API/ECFS baseline entries. + - Root/app/admin facade direct storage owner bypass scan: passed. + - Diff-added Rust risk scan: passed. + - Full PR gate: pending before PR after prerequisite PRs merge and this + branch is rebased onto `origin/main`. + +- Issue #660 API-254 current slice: + - Branch freshness check: stacked on + `overtrue/arch-storage-owner-wildcard-domain-batch` while prerequisite PRs + are pending. + - `cargo check -p rustfs --lib`: passed. + - `cargo test -p rustfs --lib storage --no-run`: passed with existing + `StorageObjectInfo` dead code warning. + - `cargo fmt --all`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - Storage owner root export wildcard scan: passed. + - Diff-added Rust risk scan: passed. + - Full PR gate: pending before PR after prerequisite PRs merge and this + branch is rebased onto `origin/main`. + +- Issue #660 API-253 current slice: + - Branch freshness check: stacked on `overtrue/arch-storage-owner-rpc-domain-batch` + while prerequisite PRs are pending. + - `cargo check -p rustfs --lib`: passed. + - `cargo test -p rustfs --lib storage --no-run`: passed with existing + `StorageObjectInfo` dead code warning. + - `cargo fmt --all`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - Storage-wide parent wildcard import scan: passed. + - Diff-added Rust risk scan: passed. + - Full PR gate: pending before PR after prerequisite PRs merge and this + branch is rebased onto `origin/main`. + +- Issue #660 API-252 current slice: + - Branch freshness check: stacked on `overtrue/arch-storage-owner-runtime-domain-batch` + while prerequisite PRs are pending. + - `cargo check -p rustfs --lib`: passed. + - `cargo test -p rustfs --lib storage::rpc --no-run`: passed. + - `cargo fmt --all`: passed. + - `bash -n scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - Storage-owner RPC wildcard import scan: passed. + - Diff-added Rust risk scan: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - Full PR gate: pending before PR after prerequisite PRs merge and this + branch is rebased onto `origin/main`. + +- Issue #660 API-251 current slice: + - Branch freshness check: stacked on `overtrue/arch-storage-owner-root-domain-batch` + while prerequisite PRs are pending. + - `cargo check -p rustfs --lib`: passed. + - `cargo test -p rustfs --lib storage::ecfs_test --no-run`: passed. + - `cargo fmt --all`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - Storage-owner runtime/helper/RPC/test root consumer scans: passed. + - Diff-added Rust risk scan: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - Full PR gate: pending before PR after prerequisite PRs merge and this + branch is rebased onto `origin/main`. + +- Issue #660 API-250 current slice: + - Branch freshness check: stacked on `overtrue/arch-root-runtime-facade-domain-batch` + while prerequisite PRs are pending. + - `cargo check -p rustfs --lib`: passed. + - `cargo fmt --all`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - Storage-owner root facade consumer scan: passed. + - Diff-added Rust risk scan: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - Full PR gate: pending before PR after prerequisite PRs merge and this + branch is rebased onto `origin/main`. + +- Issue #660 API-249 current slice: + - Branch freshness check: stacked on `overtrue/arch-root-storage-contract-domain-batch` + while prerequisite PRs are pending. + - `cargo check -p rustfs --lib`: passed. + - `cargo fmt --all`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - Root runtime facade root consumer scan: passed. + - Diff-added Rust risk scan: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - Full PR gate: pending before PR after prerequisite PRs merge and this + branch is rebased onto `origin/main`. + +- Issue #660 API-248 current slice: + - Branch freshness check: stacked on `overtrue/arch-app-usecase-contract-domain-batch` + while prerequisite PRs are pending. + - `cargo check -p rustfs --lib`: passed. + - `cargo fmt --all`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - Root storage contract root consumer scan: passed. + - Diff-added Rust risk scan: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - Full PR gate: pending before PR after prerequisite PRs merge and this + branch is rebased onto `origin/main`. + +- Issue #660 API-247 current slice: + - Branch freshness check: stacked on `overtrue/arch-admin-contract-domain-batch` + while PR #3911 is pending. + - `cargo check -p rustfs --lib`: passed. + - `cargo fmt --all`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - App usecase contract root consumer scan: passed. + - Diff-added Rust risk scan: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - Full PR gate: pending before PR after prerequisite PRs merge and this + branch is rebased onto `origin/main`. + - Issue #660 API-246 current slice: - - Branch freshness check: rebased onto current `origin/main` after PR #3911 - merged. + - Branch freshness check: stacked on PR #3911 head while PR #3911 is pending. - `cargo check -p rustfs --lib`: passed. - `cargo fmt --all`: passed. - `./scripts/check_architecture_migration_rules.sh`: passed. @@ -5933,7 +6162,8 @@ Passed before push: - Diff-added Rust risk scan: passed. - `cargo fmt --all --check`: passed. - `git diff --check`: passed. - - `make pre-pr`: passed. + - Full PR gate: pending before PR after PR #3911 merges and this branch is + rebased onto `origin/main`. - Issue #660 API-245 current slice: - Branch freshness check: rebased onto current `origin/main` after PR #3910 diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index bd825f7bc..0b67917c2 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -18,22 +18,22 @@ use std::sync::Arc; use rustfs_storage_api as storage_contracts; mod ecstore_bucket { - pub(crate) use crate::storage::ecstore_bucket::{ + pub(crate) use crate::storage::storage_api::ecstore_bucket::{ bandwidth, bucket_target_sys, lifecycle, metadata, metadata_sys, quota, replication, target, utils, versioning, versioning_sys, }; } mod ecstore_capacity { - pub(crate) use crate::storage::ecstore_capacity::is_reserved_or_invalid_bucket; + pub(crate) use crate::storage::storage_api::ecstore_capacity::is_reserved_or_invalid_bucket; } mod ecstore_client { - pub(crate) use crate::storage::ecstore_client::admin_handler_utils; + pub(crate) use crate::storage::storage_api::ecstore_client::admin_handler_utils; } pub(crate) mod ecstore_cluster { - pub(crate) use crate::storage::ecstore_cluster::{ + pub(crate) use crate::storage::storage_api::ecstore_cluster::{ ClusterDriveMembership, ClusterEndpointType, ClusterLocalNodeStorage, ClusterLocalNodeStorageSnapshot, ClusterMembershipSnapshot, ClusterNodeMembership, ClusterPeerHealth, ClusterPeerHealthSnapshot, ClusterPoolState, ClusterPoolStateSnapshot, @@ -41,34 +41,34 @@ pub(crate) mod ecstore_cluster { } mod ecstore_config { - pub(crate) use crate::storage::ecstore_config::{com, init, storageclass}; + pub(crate) use crate::storage::storage_api::ecstore_config::{com, init, storageclass}; } #[allow(unused_imports)] mod ecstore_disk { - pub(crate) use crate::storage::ecstore_disk::{RUSTFS_META_BUCKET, endpoint}; + pub(crate) use crate::storage::storage_api::ecstore_disk::{RUSTFS_META_BUCKET, endpoint}; } mod ecstore_error { - pub(crate) use crate::storage::ecstore_error::StorageError; + pub(crate) use crate::storage::storage_api::ecstore_error::StorageError; } #[allow(unused_imports)] mod ecstore_layout { - pub(crate) use crate::storage::ecstore_layout::{EndpointServerPools, Endpoints, PoolEndpoints}; + pub(crate) use crate::storage::storage_api::ecstore_layout::{EndpointServerPools, Endpoints, PoolEndpoints}; } mod ecstore_metrics { - pub(crate) use crate::storage::ecstore_metrics::{CollectMetricsOpts, MetricType, collect_local_metrics}; + pub(crate) use crate::storage::storage_api::ecstore_metrics::{CollectMetricsOpts, MetricType, collect_local_metrics}; } mod ecstore_notification { - pub(crate) use crate::storage::ecstore_notification::NotificationSys; + pub(crate) use crate::storage::storage_api::ecstore_notification::NotificationSys; } #[allow(unused_imports)] mod ecstore_rebalance { - pub(crate) use crate::storage::ecstore_rebalance::{ + pub(crate) use crate::storage::storage_api::ecstore_rebalance::{ DiskStat, RebalSaveOpt, RebalStatus, RebalanceCleanupWarningEntry, RebalanceCleanupWarnings, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord, decode_rebalance_stop_propagation_record, encode_rebalance_stop_propagation_record, @@ -76,15 +76,15 @@ mod ecstore_rebalance { } mod ecstore_rpc { - pub(crate) use crate::storage::ecstore_rpc::PeerRestClient; + pub(crate) use crate::storage::storage_api::ecstore_rpc::PeerRestClient; } mod ecstore_storage { - pub(crate) use crate::storage::ecstore_storage::ECStore; + pub(crate) use crate::storage::storage_api::ecstore_storage::ECStore; } mod ecstore_tier { - pub(crate) use crate::storage::ecstore_tier::{tier, tier_admin, tier_config, tier_handlers}; + pub(crate) use crate::storage::storage_api::ecstore_tier::{tier, tier_admin, tier_config, tier_handlers}; } pub(crate) const RUSTFS_META_BUCKET: &str = ecstore_disk::RUSTFS_META_BUCKET; @@ -414,15 +414,15 @@ pub(crate) mod data_usage { use std::sync::Arc; pub(crate) async fn load_data_usage_from_backend( - store: Arc, - ) -> Result { - crate::storage::ecstore_data_usage::load_data_usage_from_backend(store).await + store: Arc, + ) -> Result { + crate::storage::storage_api::ecstore_data_usage::load_data_usage_from_backend(store).await } } pub(crate) mod access { - pub(crate) use crate::storage::access::{ReqInfo, authorize_request}; - pub(crate) use crate::storage::request_context::{RequestContext, spawn_traced}; + pub(crate) use crate::storage::storage_api::access_consumer::{ReqInfo, authorize_request}; + pub(crate) use crate::storage::storage_api::request_context_consumer::{RequestContext, spawn_traced}; } pub(crate) mod bucket { @@ -501,7 +501,7 @@ pub(crate) mod metrics { } pub(crate) mod object { - pub(crate) use crate::storage::StorageObjectOptions; + pub(crate) use crate::storage::storage_api::StorageObjectOptions; } pub(crate) mod rebalance { diff --git a/rustfs/src/app/admin_usecase.rs b/rustfs/src/app/admin_usecase.rs index 0958ff3c6..dcdd34cb7 100644 --- a/rustfs/src/app/admin_usecase.rs +++ b/rustfs/src/app/admin_usecase.rs @@ -14,11 +14,11 @@ //! Admin application use-case contracts. -use super::storage_api::admin_usecase::StorageAdminApi; use super::storage_api::admin_usecase::admin::get_server_info; use super::storage_api::admin_usecase::capacity::{ PoolDecommissionInfo, PoolStatus, RebalStatus, get_total_usable_capacity, get_total_usable_capacity_free, }; +use super::storage_api::admin_usecase::contract::StorageAdminApi; use super::storage_api::admin_usecase::data_usage::{ apply_bucket_usage_memory_overlay, load_data_usage_from_backend, refresh_versioned_bucket_usage_from_object_layer, replace_bucket_usage_memory_from_info, diff --git a/rustfs/src/app/bucket_usecase.rs b/rustfs/src/app/bucket_usecase.rs index 44f97d5e7..774fb6a51 100644 --- a/rustfs/src/app/bucket_usecase.rs +++ b/rustfs/src/app/bucket_usecase.rs @@ -37,6 +37,12 @@ use super::storage_api::bucket_usecase::bucket::{ utils::serialize, versioning_sys::BucketVersioningSys, }; +use super::storage_api::bucket_usecase::contract::bucket::{ + BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, +}; +use super::storage_api::bucket_usecase::contract::list::{ + ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _, +}; use super::storage_api::bucket_usecase::data_usage::remove_bucket_usage_from_backend; use super::storage_api::bucket_usecase::error::StorageError; use super::storage_api::bucket_usecase::helper::{OperationHelper, spawn_background_with_context}; @@ -46,10 +52,6 @@ use super::storage_api::bucket_usecase::s3_api::bucket::{ build_list_objects_output, build_list_objects_v2_output, parse_list_object_versions_params, parse_list_objects_v2_params, rustfs_owner, }; -use super::storage_api::bucket_usecase::{ - BucketOperations, BucketOptions, DeleteBucketOptions, ListObjectVersionsInfo as StorageListObjectVersionsInfo, - ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _, MakeBucketOptions, -}; use super::storage_api::bucket_usecase::{ get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations, request_context, validate_list_object_unordered_with_delimiter, diff --git a/rustfs/src/app/capacity_dirty_scope_test.rs b/rustfs/src/app/capacity_dirty_scope_test.rs index f67672766..92a364444 100644 --- a/rustfs/src/app/capacity_dirty_scope_test.rs +++ b/rustfs/src/app/capacity_dirty_scope_test.rs @@ -13,7 +13,9 @@ // limitations under the License. use super::storage_api::test::bucket::metadata_sys; -use super::storage_api::test::{BucketOperations, BucketOptions, HealOperations as _, MakeBucketOptions, ObjectIO as _}; +use super::storage_api::test::contract::bucket::{BucketOperations, BucketOptions, MakeBucketOptions}; +use super::storage_api::test::contract::heal::HealOperations as _; +use super::storage_api::test::contract::object::ObjectIO as _; use super::storage_api::test::{ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints}; use rustfs_common::heal_channel::{HealOpts, HealScanMode}; use rustfs_object_capacity::capacity_manager::{HybridStrategyConfig, create_isolated_manager}; diff --git a/rustfs/src/app/lifecycle_transition_api_test.rs b/rustfs/src/app/lifecycle_transition_api_test.rs index 130254c1e..8aa6577df 100644 --- a/rustfs/src/app/lifecycle_transition_api_test.rs +++ b/rustfs/src/app/lifecycle_transition_api_test.rs @@ -18,13 +18,15 @@ use super::storage_api::test::bucket::{ metadata_sys, transition_api::{ReadCloser, ReaderImpl}, }; +use super::storage_api::test::contract::{ + bucket::{BucketOperations, BucketOptions, MakeBucketOptions}, + list::ListOperations as _, + multipart::MultipartOperations as _, + object::{ObjectIO as _, ObjectOperations as _}, +}; use super::storage_api::test::ecfs::FS; use super::storage_api::test::object_utils::to_s3s_etag; use super::storage_api::test::runtime::{AppWarmBackend, TierConfig, TierType, WarmBackendGetOpts}; -use super::storage_api::test::{ - BucketOperations, BucketOptions, ListOperations as _, MakeBucketOptions, MultipartOperations as _, ObjectIO as _, - ObjectOperations as _, -}; use super::storage_api::test::{ ECStore, Endpoint, EndpointServerPools, Endpoints, PoolEndpoints, StorageObjectInfo as ObjectInfo, StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader, diff --git a/rustfs/src/app/multipart_usecase.rs b/rustfs/src/app/multipart_usecase.rs index d6d738910..e44d9fe7a 100644 --- a/rustfs/src/app/multipart_usecase.rs +++ b/rustfs/src/app/multipart_usecase.rs @@ -15,8 +15,6 @@ //! Multipart application use-case contracts. use super::storage_api::multipart_usecase::ECStore; -#[cfg(test)] -use super::storage_api::multipart_usecase::HTTPPreconditions; use super::storage_api::multipart_usecase::access::has_bypass_governance_header; use super::storage_api::multipart_usecase::bucket::quota::checker::QuotaChecker; use super::storage_api::multipart_usecase::bucket::{ @@ -27,6 +25,11 @@ use super::storage_api::multipart_usecase::bucket::{ versioning_sys::BucketVersioningSys, }; use super::storage_api::multipart_usecase::compression::is_disk_compressible; +#[cfg(test)] +use super::storage_api::multipart_usecase::contract::http::HTTPPreconditions; +use super::storage_api::multipart_usecase::contract::multipart::{CompletePart, MultipartOperations as _, MultipartUploadResult}; +use super::storage_api::multipart_usecase::contract::object::{ObjectIO as _, ObjectOperations as _}; +use super::storage_api::multipart_usecase::contract::range::HTTPRangeSpec; use super::storage_api::multipart_usecase::data_usage::{ record_bucket_object_version_write_memory, record_bucket_object_write_memory, }; @@ -51,9 +54,6 @@ use super::storage_api::multipart_usecase::sse::{ extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in, map_get_object_reader_error, mark_encrypted_multipart_metadata, sse_decryption, sse_prepare_encryption, }; -use super::storage_api::multipart_usecase::{ - CompletePart, HTTPRangeSpec, MultipartOperations as _, MultipartUploadResult, ObjectIO as _, ObjectOperations as _, -}; use super::storage_api::multipart_usecase::{StorageObjectOptions as ObjectOptions, StoragePutObjReader as PutObjReader}; use crate::app::object_usecase::{build_put_like_object_lock_metadata, validate_existing_object_lock_for_write}; use crate::app::runtime_sources::{AppContext, current_app_context, resolve_object_store_handle_for_context}; diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 98c94dcd9..b455135a6 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -16,8 +16,6 @@ // Performance metrics recording (with zero-copy-metrics integration) use super::storage_api::object_usecase::ECStore; -#[cfg(test)] -use super::storage_api::object_usecase::HTTPPreconditions; use super::storage_api::object_usecase::access::{ PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut, }; @@ -50,6 +48,11 @@ use super::storage_api::object_usecase::concurrency::{ self, ConcurrencyManager, GetObjectGuard, PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, get_put_concurrency_aware_buffer_size, }; +#[cfg(test)] +use super::storage_api::object_usecase::contract::http::HTTPPreconditions; +use super::storage_api::object_usecase::contract::namespace::NamespaceLocking; +use super::storage_api::object_usecase::contract::object::{ObjectIO as _, ObjectOperations as _}; +use super::storage_api::object_usecase::contract::range::HTTPRangeSpec; use super::storage_api::object_usecase::data_usage::{ record_bucket_delete_marker_memory, record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory, @@ -78,7 +81,6 @@ use super::storage_api::object_usecase::sse::{ }; use super::storage_api::object_usecase::storage_class as storageclass; use super::storage_api::object_usecase::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper}; -use super::storage_api::object_usecase::{HTTPRangeSpec, NamespaceLocking, ObjectIO as _, ObjectOperations as _}; use super::storage_api::object_usecase::{ RFC1123, check_preconditions, get_validated_store, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, remove_object_lock_metadata_for_copy, diff --git a/rustfs/src/app/select_object.rs b/rustfs/src/app/select_object.rs index 5a0ab3c3d..f2090262c 100644 --- a/rustfs/src/app/select_object.rs +++ b/rustfs/src/app/select_object.rs @@ -1,4 +1,4 @@ -use super::storage_api::select_object::ObjectOperations as _; +use super::storage_api::select_object::contract::object::ObjectOperations as _; use super::storage_api::select_object::options::get_opts; use super::storage_api::select_object::request_context::spawn_traced; use super::storage_api::select_object::{get_validated_store, validate_sse_headers_for_read, validate_ssec_for_read}; diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index f5fe33b60..3f5720f78 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -18,27 +18,29 @@ use rustfs_storage_api as storage_contracts; #[cfg(test)] #[allow(non_snake_case)] -pub(crate) fn EndpointServerPools(pools: Vec) -> crate::storage::EndpointServerPools { - crate::storage::EndpointServerPools::from(pools) +pub(crate) fn EndpointServerPools( + pools: Vec, +) -> crate::storage::storage_api::EndpointServerPools { + crate::storage::storage_api::EndpointServerPools::from(pools) } pub(crate) mod admin { pub(crate) async fn get_server_info(get_pools: bool) -> rustfs_madmin::InfoMessage { - crate::storage::ecstore_admin::get_server_info(get_pools).await + crate::storage::storage_api::ecstore_admin::get_server_info(get_pools).await } } pub(crate) mod capacity { - pub(crate) type PoolDecommissionInfo = crate::storage::ecstore_capacity::PoolDecommissionInfo; - pub(crate) type PoolStatus = crate::storage::ecstore_capacity::PoolStatus; - pub(crate) type RebalStatus = crate::storage::ecstore_rebalance::RebalStatus; + pub(crate) type PoolDecommissionInfo = crate::storage::storage_api::ecstore_capacity::PoolDecommissionInfo; + pub(crate) type PoolStatus = crate::storage::storage_api::ecstore_capacity::PoolStatus; + pub(crate) type RebalStatus = crate::storage::storage_api::ecstore_rebalance::RebalStatus; pub(crate) fn get_total_usable_capacity(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize { - crate::storage::ecstore_capacity::get_total_usable_capacity(disks, info) + crate::storage::storage_api::ecstore_capacity::get_total_usable_capacity(disks, info) } pub(crate) fn get_total_usable_capacity_free(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize { - crate::storage::ecstore_capacity::get_total_usable_capacity_free(disks, info) + crate::storage::storage_api::ecstore_capacity::get_total_usable_capacity_free(disks, info) } } @@ -46,33 +48,38 @@ pub(crate) mod data_usage { use std::sync::Arc; pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) { - crate::storage::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await; + crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await; } pub(crate) async fn load_data_usage_from_backend( - store: Arc, - ) -> Result { - crate::storage::ecstore_data_usage::load_data_usage_from_backend(store).await + store: Arc, + ) -> Result { + crate::storage::storage_api::ecstore_data_usage::load_data_usage_from_backend(store).await } pub(crate) async fn refresh_versioned_bucket_usage_from_object_layer( - store: Arc, + store: Arc, data_usage_info: &mut rustfs_data_usage::DataUsageInfo, ) { - crate::storage::ecstore_data_usage::refresh_versioned_bucket_usage_from_object_layer(store, data_usage_info).await; - } - - pub(crate) async fn replace_bucket_usage_memory_from_info(data_usage_info: &rustfs_data_usage::DataUsageInfo) { - crate::storage::ecstore_data_usage::replace_bucket_usage_memory_from_info(data_usage_info).await; - } - - pub(crate) async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) { - crate::storage::ecstore_data_usage::record_bucket_object_delete_memory(bucket, deleted_size, removed_current_object) + crate::storage::storage_api::ecstore_data_usage::refresh_versioned_bucket_usage_from_object_layer(store, data_usage_info) .await; } + pub(crate) async fn replace_bucket_usage_memory_from_info(data_usage_info: &rustfs_data_usage::DataUsageInfo) { + crate::storage::storage_api::ecstore_data_usage::replace_bucket_usage_memory_from_info(data_usage_info).await; + } + + pub(crate) async fn record_bucket_object_delete_memory(bucket: &str, deleted_size: u64, removed_current_object: bool) { + crate::storage::storage_api::ecstore_data_usage::record_bucket_object_delete_memory( + bucket, + deleted_size, + removed_current_object, + ) + .await; + } + pub(crate) async fn record_bucket_delete_marker_memory(bucket: &str) { - crate::storage::ecstore_data_usage::record_bucket_delete_marker_memory(bucket).await; + crate::storage::storage_api::ecstore_data_usage::record_bucket_delete_marker_memory(bucket).await; } pub(crate) async fn record_bucket_object_version_write_memory( @@ -80,111 +87,120 @@ pub(crate) mod data_usage { previous_current_size: Option, new_size: u64, ) { - crate::storage::ecstore_data_usage::record_bucket_object_version_write_memory(bucket, previous_current_size, new_size) - .await; + crate::storage::storage_api::ecstore_data_usage::record_bucket_object_version_write_memory( + bucket, + previous_current_size, + new_size, + ) + .await; } pub(crate) async fn record_bucket_object_write_memory(bucket: &str, previous_current_size: Option, new_size: u64) { - crate::storage::ecstore_data_usage::record_bucket_object_write_memory(bucket, previous_current_size, new_size).await; + crate::storage::storage_api::ecstore_data_usage::record_bucket_object_write_memory( + bucket, + previous_current_size, + new_size, + ) + .await; } pub(crate) async fn remove_bucket_usage_from_backend( - store: Arc, + store: Arc, bucket: &str, - ) -> Result<(), crate::storage::StorageError> { - crate::storage::ecstore_data_usage::remove_bucket_usage_from_backend(store, bucket).await + ) -> Result<(), crate::storage::storage_api::StorageError> { + crate::storage::storage_api::ecstore_data_usage::remove_bucket_usage_from_backend(store, bucket).await } } pub(crate) mod runtime { use std::{collections::HashMap, sync::Arc}; - pub(crate) type BucketBandwidthMonitor = crate::storage::BucketBandwidthMonitor; - pub(crate) type DailyAllTierStats = crate::storage::DailyAllTierStats; - pub(crate) type DynReplicationPool = crate::storage::DynReplicationPool; - pub(crate) type ExpiryState = crate::storage::ExpiryState; - pub(crate) type NotificationSys = crate::storage::NotificationSys; - pub(crate) type ObjectStoreResolver = crate::storage::ObjectStoreResolver; - pub(crate) type ReplicationStats = crate::storage::ReplicationStats; + pub(crate) type BucketBandwidthMonitor = crate::storage::storage_api::BucketBandwidthMonitor; + pub(crate) type DailyAllTierStats = crate::storage::storage_api::DailyAllTierStats; + pub(crate) type DynReplicationPool = crate::storage::storage_api::DynReplicationPool; + pub(crate) type ExpiryState = crate::storage::storage_api::ExpiryState; + pub(crate) type NotificationSys = crate::storage::storage_api::NotificationSys; + pub(crate) type ObjectStoreResolver = crate::storage::storage_api::ObjectStoreResolver; + pub(crate) type ReplicationStats = crate::storage::storage_api::ReplicationStats; pub(crate) type ScannerMetricsReport = rustfs_common::metrics::ScannerMetricsReport; - pub(crate) type StorageClassConfig = crate::storage::ecstore_config::storageclass::Config; - pub(crate) type TierConfigMgr = crate::storage::TierConfigMgr; + pub(crate) type StorageClassConfig = crate::storage::storage_api::ecstore_config::storageclass::Config; + pub(crate) type TierConfigMgr = crate::storage::storage_api::TierConfigMgr; #[cfg(test)] - pub(crate) type TierConfig = crate::storage::ecstore_tier::tier_config::TierConfig; + pub(crate) type TierConfig = crate::storage::storage_api::ecstore_tier::tier_config::TierConfig; #[cfg(test)] - pub(crate) type TierType = crate::storage::ecstore_tier::tier_config::TierType; + pub(crate) type TierType = crate::storage::storage_api::ecstore_tier::tier_config::TierType; #[cfg(test)] - pub(crate) use crate::storage::ecstore_tier::warm_backend::WarmBackend as AppWarmBackend; + pub(crate) use crate::storage::storage_api::ecstore_tier::warm_backend::WarmBackend as AppWarmBackend; #[cfg(test)] - pub(crate) type WarmBackendGetOpts = crate::storage::ecstore_tier::warm_backend::WarmBackendGetOpts; + pub(crate) type WarmBackendGetOpts = crate::storage::storage_api::ecstore_tier::warm_backend::WarmBackendGetOpts; pub(crate) fn set_global_storage_class(cfg: StorageClassConfig) { - crate::storage::ecstore_config::set_global_storage_class(cfg); + crate::storage::storage_api::ecstore_config::set_global_storage_class(cfg); } - pub(crate) fn get_global_endpoints_opt() -> Option { - crate::storage::get_global_endpoints_opt() + pub(crate) fn get_global_endpoints_opt() -> Option { + crate::storage::storage_api::get_global_endpoints_opt() } pub(crate) fn get_global_deployment_id() -> Option { - crate::storage::get_global_deployment_id() + crate::storage::storage_api::get_global_deployment_id() } pub(crate) fn get_global_lock_client() -> Option> { - crate::storage::get_global_lock_client() + crate::storage::storage_api::get_global_lock_client() } pub(crate) fn get_global_lock_clients() -> Option<&'static HashMap>> { - crate::storage::get_global_lock_clients() + crate::storage::storage_api::get_global_lock_clients() } pub(crate) fn get_global_region() -> Option { - crate::storage::get_global_region() + crate::storage::storage_api::get_global_region() } pub(crate) fn global_rustfs_port() -> u16 { - crate::storage::global_rustfs_port() + crate::storage::storage_api::global_rustfs_port() } pub(crate) fn get_global_tier_config_mgr() -> Arc> { - crate::storage::get_global_tier_config_mgr() + crate::storage::storage_api::get_global_tier_config_mgr() } pub(crate) fn get_global_expiry_state() -> Arc> { - crate::storage::get_global_expiry_state() + crate::storage::storage_api::get_global_expiry_state() } - pub(crate) fn new_object_layer_fn() -> Option> { - crate::storage::new_object_layer_fn() + pub(crate) fn new_object_layer_fn() -> Option> { + crate::storage::storage_api::new_object_layer_fn() } pub(crate) fn set_object_store_resolver(resolver: Arc) -> bool { - crate::storage::set_object_store_resolver(resolver) + crate::storage::storage_api::set_object_store_resolver(resolver) } pub(crate) fn get_global_notification_sys() -> Option<&'static NotificationSys> { - crate::storage::get_global_notification_sys() + crate::storage::storage_api::get_global_notification_sys() } pub(crate) fn get_global_bucket_monitor() -> Option> { - crate::storage::get_global_bucket_monitor() + crate::storage::storage_api::get_global_bucket_monitor() } pub(crate) fn get_global_replication_pool() -> Option> { - crate::storage::get_global_replication_pool() + crate::storage::storage_api::get_global_replication_pool() } pub(crate) fn get_global_replication_stats() -> Option> { - crate::storage::get_global_replication_stats() + crate::storage::storage_api::get_global_replication_stats() } pub(crate) fn get_global_boot_time() -> Option { - crate::storage::get_global_boot_time() + crate::storage::storage_api::get_global_boot_time() } pub(crate) fn get_daily_all_tier_stats() -> DailyAllTierStats { - crate::storage::get_daily_all_tier_stats() + crate::storage::storage_api::get_daily_all_tier_stats() } pub(crate) async fn collect_scanner_metrics_report() -> ScannerMetricsReport { @@ -193,14 +209,14 @@ pub(crate) mod runtime { #[cfg(test)] pub(crate) async fn init_local_disks( - endpoint_pools: crate::storage::EndpointServerPools, - ) -> Result<(), crate::storage::StorageError> { - crate::storage::init_local_disks(endpoint_pools).await + endpoint_pools: crate::storage::storage_api::EndpointServerPools, + ) -> Result<(), crate::storage::storage_api::StorageError> { + crate::storage::storage_api::init_local_disks(endpoint_pools).await } } pub(crate) mod access { - pub(crate) use crate::storage::access::{ + pub(crate) use crate::storage::storage_api::access_consumer::{ PostObjectRequestMarker, ReqInfo, authorize_request, has_bypass_governance_header, req_info_mut, req_info_ref, }; } @@ -214,7 +230,7 @@ pub(crate) mod bucket { impl ObjectLockConfigExt for s3s::dto::ObjectLockConfiguration { fn enabled(&self) -> bool { - ::enabled(self) + ::enabled(self) } } @@ -225,13 +241,13 @@ pub(crate) mod bucket { impl ReplicationConfigExt for s3s::dto::ReplicationConfiguration { fn filter_target_arns(&self, obj: &replication::ObjectOpts) -> Vec { - ::filter_target_arns( + ::filter_target_arns( self, obj, ) } fn replicate(&self, opts: &replication::ObjectOpts) -> bool { - ::replicate( + ::replicate( self, opts, ) } @@ -245,17 +261,19 @@ pub(crate) mod bucket { impl VersioningConfigExt for s3s::dto::VersioningConfiguration { fn enabled(&self) -> bool { - ::enabled(self) + ::enabled( + self, + ) } fn prefix_enabled(&self, prefix: &str) -> bool { - ::prefix_enabled( + ::prefix_enabled( self, prefix, ) } fn suspended(&self) -> bool { - ::suspended(self) + ::suspended(self) } } @@ -263,25 +281,26 @@ pub(crate) mod bucket { lifecycle: &s3s::dto::BucketLifecycleConfiguration, obj: &lifecycle::lifecycle::ObjectOpts, ) -> lifecycle::lifecycle::Event { - crate::storage::ecstore_bucket::lifecycle::lifecycle::Lifecycle::predict_expiration(lifecycle, obj).await + crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::Lifecycle::predict_expiration(lifecycle, obj).await } pub(crate) fn validate_restore_request( request: &s3s::dto::RestoreRequest, - api: Arc, + api: Arc, ) -> std::io::Result<()> { - ::validate( + ::validate( request, api, ) } pub(crate) mod bucket_target_sys { - pub(crate) type BucketTargetSys = crate::storage::ecstore_bucket::bucket_target_sys::BucketTargetSys; + pub(crate) type BucketTargetSys = crate::storage::storage_api::ecstore_bucket::bucket_target_sys::BucketTargetSys; } pub(crate) mod lifecycle { pub(crate) mod bucket_lifecycle_audit { - pub(crate) type LcEventSrc = crate::storage::ecstore_bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; + pub(crate) type LcEventSrc = + crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc; } pub(crate) mod bucket_lifecycle_ops { @@ -290,52 +309,62 @@ pub(crate) mod bucket { use super::bucket_lifecycle_audit::LcEventSrc; #[cfg(test)] - pub(crate) async fn init_background_expiry(api: Arc) { - crate::storage::ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(api).await; + pub(crate) async fn init_background_expiry(api: Arc) { + crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::init_background_expiry(api).await; } pub(crate) async fn enqueue_expiry_for_existing_objects( - api: Arc, + api: Arc, bucket: &str, - ) -> Result<(), crate::storage::StorageError> { - crate::storage::ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_expiry_for_existing_objects(api, bucket) - .await - } - - pub(crate) async fn enqueue_transition_for_existing_objects( - api: Arc, - bucket: &str, - ) -> Result<(), crate::storage::StorageError> { - crate::storage::ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( + ) -> Result<(), crate::storage::storage_api::StorageError> { + crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_expiry_for_existing_objects( api, bucket, ) .await } - pub(crate) async fn enqueue_transition_immediate(oi: &crate::storage::StorageObjectInfo, src: LcEventSrc) { - crate::storage::ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_immediate(oi, src).await; + pub(crate) async fn enqueue_transition_for_existing_objects( + api: Arc, + bucket: &str, + ) -> Result<(), crate::storage::storage_api::StorageError> { + crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_for_existing_objects( + api, bucket, + ) + .await + } + + pub(crate) async fn enqueue_transition_immediate( + oi: &crate::storage::storage_api::StorageObjectInfo, + src: LcEventSrc, + ) { + crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::enqueue_transition_immediate( + oi, src, + ) + .await; } pub(crate) async fn post_restore_opts( version_id: &str, bucket: &str, object: &str, - ) -> Result { - crate::storage::ecstore_bucket::lifecycle::bucket_lifecycle_ops::post_restore_opts(version_id, bucket, object) - .await + ) -> Result { + crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::post_restore_opts( + version_id, bucket, object, + ) + .await } pub(crate) async fn validate_transition_tier( lc: &s3s::dto::BucketLifecycleConfiguration, ) -> Result<(), std::io::Error> { - crate::storage::ecstore_bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier(lc).await + crate::storage::storage_api::ecstore_bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier(lc).await } pub(crate) async fn validate_lifecycle_config( lc: &s3s::dto::BucketLifecycleConfiguration, lock_config: &s3s::dto::ObjectLockConfiguration, ) -> Result<(), std::io::Error> { - use crate::storage::ecstore_bucket::lifecycle::lifecycle::Lifecycle as _; + use crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::Lifecycle as _; lc.validate(lock_config).await } @@ -343,16 +372,17 @@ pub(crate) mod bucket { pub(crate) mod lifecycle_contract { #[cfg(test)] - pub(crate) type IlmAction = crate::storage::ecstore_bucket::lifecycle::lifecycle::IlmAction; - pub(crate) type Event = crate::storage::ecstore_bucket::lifecycle::lifecycle::Event; - pub(crate) type ObjectOpts = crate::storage::ecstore_bucket::lifecycle::lifecycle::ObjectOpts; - pub(crate) type TransitionOptions = crate::storage::ecstore_bucket::lifecycle::lifecycle::TransitionOptions; + pub(crate) type IlmAction = crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::IlmAction; + pub(crate) type Event = crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::Event; + pub(crate) type ObjectOpts = crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::ObjectOpts; + pub(crate) type TransitionOptions = + crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::TransitionOptions; pub(crate) const TRANSITION_COMPLETE: &str = - crate::storage::ecstore_bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; + crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::TRANSITION_COMPLETE; pub(crate) fn expected_expiry_time(mod_time: time::OffsetDateTime, days: i32) -> time::OffsetDateTime { - crate::storage::ecstore_bucket::lifecycle::lifecycle::expected_expiry_time(mod_time, days) + crate::storage::storage_api::ecstore_bucket::lifecycle::lifecycle::expected_expiry_time(mod_time, days) } } pub(crate) use lifecycle_contract as lifecycle; @@ -361,15 +391,18 @@ pub(crate) mod bucket { use std::sync::Arc; pub(crate) async fn persist_tier_delete_journal_entry( - api: Arc, + api: Arc, je: &super::tier_sweeper::Jentry, ) -> std::io::Result<()> { - crate::storage::ecstore_bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry(api, je).await + crate::storage::storage_api::ecstore_bucket::lifecycle::tier_delete_journal::persist_tier_delete_journal_entry( + api, je, + ) + .await } } pub(crate) mod tier_sweeper { - pub(crate) type Jentry = crate::storage::ecstore_bucket::lifecycle::tier_sweeper::Jentry; + pub(crate) type Jentry = crate::storage::storage_api::ecstore_bucket::lifecycle::tier_sweeper::Jentry; pub(crate) fn transitioned_delete_journal_entry( version_id: Option, @@ -377,7 +410,7 @@ pub(crate) mod bucket { suspended: bool, transitioned: &super::super::super::storage_contracts::TransitionedObject, ) -> Option { - crate::storage::ecstore_bucket::lifecycle::tier_sweeper::transitioned_delete_journal_entry( + crate::storage::storage_api::ecstore_bucket::lifecycle::tier_sweeper::transitioned_delete_journal_entry( version_id, versioned, suspended, @@ -388,25 +421,32 @@ pub(crate) mod bucket { pub(crate) fn transitioned_force_delete_journal_entry( transitioned: &super::super::super::storage_contracts::TransitionedObject, ) -> Option { - crate::storage::ecstore_bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(transitioned) + crate::storage::storage_api::ecstore_bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry( + transitioned, + ) } } } pub(crate) mod metadata { - pub(crate) const BUCKET_CORS_CONFIG: &str = crate::storage::ecstore_bucket::metadata::BUCKET_CORS_CONFIG; - pub(crate) const BUCKET_LIFECYCLE_CONFIG: &str = crate::storage::ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG; - pub(crate) const BUCKET_NOTIFICATION_CONFIG: &str = crate::storage::ecstore_bucket::metadata::BUCKET_NOTIFICATION_CONFIG; - pub(crate) const BUCKET_POLICY_CONFIG: &str = crate::storage::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG; + pub(crate) const BUCKET_CORS_CONFIG: &str = crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_CORS_CONFIG; + pub(crate) const BUCKET_LIFECYCLE_CONFIG: &str = + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_LIFECYCLE_CONFIG; + pub(crate) const BUCKET_NOTIFICATION_CONFIG: &str = + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_NOTIFICATION_CONFIG; + pub(crate) const BUCKET_POLICY_CONFIG: &str = crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_POLICY_CONFIG; pub(crate) const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = - crate::storage::ecstore_bucket::metadata::BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG; - pub(crate) const BUCKET_REPLICATION_CONFIG: &str = crate::storage::ecstore_bucket::metadata::BUCKET_REPLICATION_CONFIG; - pub(crate) const BUCKET_SSECONFIG: &str = crate::storage::ecstore_bucket::metadata::BUCKET_SSECONFIG; - pub(crate) const BUCKET_TAGGING_CONFIG: &str = crate::storage::ecstore_bucket::metadata::BUCKET_TAGGING_CONFIG; - pub(crate) const BUCKET_TARGETS_FILE: &str = crate::storage::ecstore_bucket::metadata::BUCKET_TARGETS_FILE; - pub(crate) const BUCKET_VERSIONING_CONFIG: &str = crate::storage::ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG; + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG; + pub(crate) const BUCKET_REPLICATION_CONFIG: &str = + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_REPLICATION_CONFIG; + pub(crate) const BUCKET_SSECONFIG: &str = crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_SSECONFIG; + pub(crate) const BUCKET_TAGGING_CONFIG: &str = + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_TAGGING_CONFIG; + pub(crate) const BUCKET_TARGETS_FILE: &str = crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_TARGETS_FILE; + pub(crate) const BUCKET_VERSIONING_CONFIG: &str = + crate::storage::storage_api::ecstore_bucket::metadata::BUCKET_VERSIONING_CONFIG; #[cfg(test)] - pub(crate) const OBJECT_LOCK_CONFIG: &str = crate::storage::ecstore_bucket::metadata::OBJECT_LOCK_CONFIG; + pub(crate) const OBJECT_LOCK_CONFIG: &str = crate::storage::storage_api::ecstore_bucket::metadata::OBJECT_LOCK_CONFIG; } pub(crate) mod metadata_sys { @@ -422,89 +462,96 @@ pub(crate) mod bucket { use super::target::BucketTargets; - pub(crate) type BucketMetadataSys = crate::storage::ecstore_bucket::metadata_sys::BucketMetadataSys; + pub(crate) type BucketMetadataSys = crate::storage::storage_api::ecstore_bucket::metadata_sys::BucketMetadataSys; #[cfg(test)] - pub(crate) async fn init_bucket_metadata_sys(api: Arc, buckets: Vec) { - crate::storage::ecstore_bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await; + pub(crate) async fn init_bucket_metadata_sys(api: Arc, buckets: Vec) { + crate::storage::storage_api::ecstore_bucket::metadata_sys::init_bucket_metadata_sys(api, buckets).await; } - pub(crate) async fn delete(bucket: &str, config_file: &str) -> Result { - crate::storage::ecstore_bucket::metadata_sys::delete(bucket, config_file).await + pub(crate) async fn delete( + bucket: &str, + config_file: &str, + ) -> Result { + crate::storage::storage_api::ecstore_bucket::metadata_sys::delete(bucket, config_file).await } pub(crate) async fn get_bucket_policy( bucket: &str, - ) -> Result<(BucketPolicy, OffsetDateTime), crate::storage::StorageError> { - crate::storage::ecstore_bucket::metadata_sys::get_bucket_policy(bucket).await + ) -> Result<(BucketPolicy, OffsetDateTime), crate::storage::storage_api::StorageError> { + crate::storage::storage_api::ecstore_bucket::metadata_sys::get_bucket_policy(bucket).await } pub(crate) async fn get_bucket_policy_raw( bucket: &str, - ) -> Result<(String, OffsetDateTime), crate::storage::StorageError> { - crate::storage::ecstore_bucket::metadata_sys::get_bucket_policy_raw(bucket).await + ) -> Result<(String, OffsetDateTime), crate::storage::storage_api::StorageError> { + crate::storage::storage_api::ecstore_bucket::metadata_sys::get_bucket_policy_raw(bucket).await } - pub(crate) async fn get_bucket_targets_config(bucket: &str) -> Result { - crate::storage::ecstore_bucket::metadata_sys::get_bucket_targets_config(bucket).await + pub(crate) async fn get_bucket_targets_config( + bucket: &str, + ) -> Result { + crate::storage::storage_api::ecstore_bucket::metadata_sys::get_bucket_targets_config(bucket).await } pub(crate) async fn get_cors_config( bucket: &str, - ) -> Result<(CORSConfiguration, OffsetDateTime), crate::storage::StorageError> { - crate::storage::ecstore_bucket::metadata_sys::get_cors_config(bucket).await + ) -> Result<(CORSConfiguration, OffsetDateTime), crate::storage::storage_api::StorageError> { + crate::storage::storage_api::ecstore_bucket::metadata_sys::get_cors_config(bucket).await } pub(crate) fn get_global_bucket_metadata_sys() -> Option>> { - crate::storage::ecstore_bucket::metadata_sys::get_global_bucket_metadata_sys() + crate::storage::storage_api::ecstore_bucket::metadata_sys::get_global_bucket_metadata_sys() } pub(crate) async fn get_lifecycle_config( bucket: &str, - ) -> Result<(BucketLifecycleConfiguration, OffsetDateTime), crate::storage::StorageError> { - crate::storage::ecstore_bucket::metadata_sys::get_lifecycle_config(bucket).await + ) -> Result<(BucketLifecycleConfiguration, OffsetDateTime), crate::storage::storage_api::StorageError> { + crate::storage::storage_api::ecstore_bucket::metadata_sys::get_lifecycle_config(bucket).await } pub(crate) async fn get_notification_config( bucket: &str, - ) -> Result, crate::storage::StorageError> { - crate::storage::ecstore_bucket::metadata_sys::get_notification_config(bucket).await + ) -> Result, crate::storage::storage_api::StorageError> { + crate::storage::storage_api::ecstore_bucket::metadata_sys::get_notification_config(bucket).await } pub(crate) async fn get_object_lock_config( bucket: &str, - ) -> Result<(ObjectLockConfiguration, OffsetDateTime), crate::storage::StorageError> { - crate::storage::ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await + ) -> Result<(ObjectLockConfiguration, OffsetDateTime), crate::storage::storage_api::StorageError> { + crate::storage::storage_api::ecstore_bucket::metadata_sys::get_object_lock_config(bucket).await } pub(crate) async fn get_public_access_block_config( bucket: &str, - ) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime), crate::storage::StorageError> { - crate::storage::ecstore_bucket::metadata_sys::get_public_access_block_config(bucket).await + ) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime), crate::storage::storage_api::StorageError> { + crate::storage::storage_api::ecstore_bucket::metadata_sys::get_public_access_block_config(bucket).await } pub(crate) async fn get_replication_config( bucket: &str, - ) -> Result<(s3s::dto::ReplicationConfiguration, OffsetDateTime), crate::storage::StorageError> { - crate::storage::ecstore_bucket::metadata_sys::get_replication_config(bucket).await + ) -> Result<(s3s::dto::ReplicationConfiguration, OffsetDateTime), crate::storage::storage_api::StorageError> { + crate::storage::storage_api::ecstore_bucket::metadata_sys::get_replication_config(bucket).await } pub(crate) async fn get_sse_config( bucket: &str, - ) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime), crate::storage::StorageError> { - crate::storage::ecstore_bucket::metadata_sys::get_sse_config(bucket).await + ) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime), crate::storage::storage_api::StorageError> { + crate::storage::storage_api::ecstore_bucket::metadata_sys::get_sse_config(bucket).await } - pub(crate) async fn get_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime), crate::storage::StorageError> { - crate::storage::ecstore_bucket::metadata_sys::get_tagging_config(bucket).await + pub(crate) async fn get_tagging_config( + bucket: &str, + ) -> Result<(Tagging, OffsetDateTime), crate::storage::storage_api::StorageError> { + crate::storage::storage_api::ecstore_bucket::metadata_sys::get_tagging_config(bucket).await } pub(crate) async fn update( bucket: &str, config_file: &str, data: Vec, - ) -> Result { - crate::storage::ecstore_bucket::metadata_sys::update(bucket, config_file, data).await + ) -> Result { + crate::storage::storage_api::ecstore_bucket::metadata_sys::update(bucket, config_file, data).await } } @@ -513,28 +560,28 @@ pub(crate) mod bucket { pub(crate) fn get_object_legalhold_meta( meta: &std::collections::HashMap, ) -> s3s::dto::ObjectLockLegalHold { - crate::storage::ecstore_bucket::object_lock::objectlock::get_object_legalhold_meta(meta) + crate::storage::storage_api::ecstore_bucket::object_lock::objectlock::get_object_legalhold_meta(meta) } pub(crate) fn get_object_retention_meta( meta: &std::collections::HashMap, ) -> s3s::dto::ObjectLockRetention { - crate::storage::ecstore_bucket::object_lock::objectlock::get_object_retention_meta(meta) + crate::storage::storage_api::ecstore_bucket::object_lock::objectlock::get_object_retention_meta(meta) } } pub(crate) mod objectlock_sys { pub(crate) type BucketObjectLockSys = - crate::storage::ecstore_bucket::object_lock::objectlock_sys::BucketObjectLockSys; + crate::storage::storage_api::ecstore_bucket::object_lock::objectlock_sys::BucketObjectLockSys; pub(crate) type ObjectLockBlockReason = - crate::storage::ecstore_bucket::object_lock::objectlock_sys::ObjectLockBlockReason; + crate::storage::storage_api::ecstore_bucket::object_lock::objectlock_sys::ObjectLockBlockReason; pub(crate) async fn check_object_lock_for_deletion( bucket: &str, - obj_info: &crate::storage::StorageObjectInfo, + obj_info: &crate::storage::storage_api::StorageObjectInfo, bypass_governance: bool, ) -> Option { - crate::storage::ecstore_bucket::object_lock::objectlock_sys::check_object_lock_for_deletion( + crate::storage::storage_api::ecstore_bucket::object_lock::objectlock_sys::check_object_lock_for_deletion( bucket, obj_info, bypass_governance, @@ -543,40 +590,45 @@ pub(crate) mod bucket { } pub(crate) fn is_retention_active(mode: &str, retain_until_date: Option<&s3s::dto::Date>) -> bool { - crate::storage::ecstore_bucket::object_lock::objectlock_sys::is_retention_active(mode, retain_until_date) + crate::storage::storage_api::ecstore_bucket::object_lock::objectlock_sys::is_retention_active( + mode, + retain_until_date, + ) } } } pub(crate) mod policy_sys { - pub(crate) type PolicySys = crate::storage::ecstore_bucket::policy_sys::PolicySys; + pub(crate) type PolicySys = crate::storage::storage_api::ecstore_bucket::policy_sys::PolicySys; } pub(crate) mod quota { pub(crate) mod checker { - pub(crate) type QuotaChecker = crate::storage::ecstore_bucket::quota::checker::QuotaChecker; + pub(crate) type QuotaChecker = crate::storage::storage_api::ecstore_bucket::quota::checker::QuotaChecker; } - pub(crate) type QuotaOperation = crate::storage::ecstore_bucket::quota::QuotaOperation; + pub(crate) type QuotaOperation = crate::storage::storage_api::ecstore_bucket::quota::QuotaOperation; } pub(crate) mod replication { use std::collections::HashMap; use std::sync::Arc; - pub(crate) type DeletedObjectReplicationInfo = crate::storage::ecstore_bucket::replication::DeletedObjectReplicationInfo; - pub(crate) type MustReplicateOptions = crate::storage::ecstore_bucket::replication::MustReplicateOptions; - pub(crate) type ObjectOpts = crate::storage::ecstore_bucket::replication::ObjectOpts; + pub(crate) type DeletedObjectReplicationInfo = + crate::storage::storage_api::ecstore_bucket::replication::DeletedObjectReplicationInfo; + pub(crate) type MustReplicateOptions = crate::storage::storage_api::ecstore_bucket::replication::MustReplicateOptions; + pub(crate) type ObjectOpts = crate::storage::storage_api::ecstore_bucket::replication::ObjectOpts; pub(crate) type ReplicateDecision = rustfs_filemeta::ReplicateDecision; pub(crate) async fn check_replicate_delete( bucket: &str, dobj: &super::super::storage_contracts::ObjectToDelete, - oi: &crate::storage::StorageObjectInfo, - del_opts: &crate::storage::StorageObjectOptions, + oi: &crate::storage::storage_api::StorageObjectInfo, + del_opts: &crate::storage::storage_api::StorageObjectOptions, gerr: Option, ) -> ReplicateDecision { - crate::storage::ecstore_bucket::replication::check_replicate_delete(bucket, dobj, oi, del_opts, gerr).await + crate::storage::storage_api::ecstore_bucket::replication::check_replicate_delete(bucket, dobj, oi, del_opts, gerr) + .await } pub(crate) fn get_must_replicate_options( @@ -584,9 +636,9 @@ pub(crate) mod bucket { user_tags: String, status: rustfs_filemeta::ReplicationStatusType, op_type: rustfs_filemeta::ReplicationType, - opts: crate::storage::StorageObjectOptions, + opts: crate::storage::storage_api::StorageObjectOptions, ) -> MustReplicateOptions { - crate::storage::ecstore_bucket::replication::get_must_replicate_options( + crate::storage::storage_api::ecstore_bucket::replication::get_must_replicate_options( user_defined, user_tags, status, @@ -596,76 +648,76 @@ pub(crate) mod bucket { } pub(crate) async fn must_replicate(bucket: &str, object: &str, mopts: MustReplicateOptions) -> ReplicateDecision { - crate::storage::ecstore_bucket::replication::must_replicate(bucket, object, mopts).await + crate::storage::storage_api::ecstore_bucket::replication::must_replicate(bucket, object, mopts).await } pub(crate) async fn schedule_replication( - oi: crate::storage::StorageObjectInfo, - store: Arc, + oi: crate::storage::storage_api::StorageObjectInfo, + store: Arc, dsc: ReplicateDecision, op_type: rustfs_filemeta::ReplicationType, ) { - crate::storage::ecstore_bucket::replication::schedule_replication(oi, store, dsc, op_type).await; + crate::storage::storage_api::ecstore_bucket::replication::schedule_replication(oi, store, dsc, op_type).await; } pub(crate) async fn schedule_replication_delete(dv: DeletedObjectReplicationInfo) { - crate::storage::ecstore_bucket::replication::schedule_replication_delete(dv).await; + crate::storage::storage_api::ecstore_bucket::replication::schedule_replication_delete(dv).await; } } pub(crate) mod tagging { pub(crate) fn decode_tags(tags: &str) -> Vec { - crate::storage::ecstore_bucket::tagging::decode_tags(tags) + crate::storage::storage_api::ecstore_bucket::tagging::decode_tags(tags) } } pub(crate) mod target { #[cfg(test)] - pub(crate) type BucketTarget = crate::storage::ecstore_bucket::target::BucketTarget; - pub(crate) type BucketTargetType = crate::storage::ecstore_bucket::target::BucketTargetType; - pub(crate) type BucketTargets = crate::storage::ecstore_bucket::target::BucketTargets; + pub(crate) type BucketTarget = crate::storage::storage_api::ecstore_bucket::target::BucketTarget; + pub(crate) type BucketTargetType = crate::storage::storage_api::ecstore_bucket::target::BucketTargetType; + pub(crate) type BucketTargets = crate::storage::storage_api::ecstore_bucket::target::BucketTargets; } pub(crate) mod utils { pub(crate) fn serialize(val: &T) -> s3s::xml::SerResult> { - crate::storage::ecstore_bucket::utils::serialize(val) + crate::storage::storage_api::ecstore_bucket::utils::serialize(val) } } #[cfg(test)] pub(crate) mod transition_api { - pub(crate) type ReadCloser = crate::storage::ecstore_client::transition_api::ReadCloser; - pub(crate) type ReaderImpl = crate::storage::ecstore_client::transition_api::ReaderImpl; + pub(crate) type ReadCloser = crate::storage::storage_api::ecstore_client::transition_api::ReadCloser; + pub(crate) type ReaderImpl = crate::storage::storage_api::ecstore_client::transition_api::ReaderImpl; } pub(crate) mod versioning_sys { - pub(crate) type BucketVersioningSys = crate::storage::ecstore_bucket::versioning_sys::BucketVersioningSys; + pub(crate) type BucketVersioningSys = crate::storage::storage_api::ecstore_bucket::versioning_sys::BucketVersioningSys; } } pub(crate) mod concurrency { - pub(crate) use crate::storage::concurrency::{ + pub(crate) use crate::storage::storage_api::concurrency_consumer::{ ConcurrencyManager, GetObjectGuard, IoQueueStatus, IoStrategy, PutObjectGuard, get_concurrency_aware_buffer_size, get_concurrency_manager, get_put_concurrency_aware_buffer_size, }; } pub(crate) mod compression { - pub(crate) use crate::storage::ecstore_compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; + pub(crate) use crate::storage::storage_api::ecstore_compression::{MIN_DISK_COMPRESSIBLE_SIZE, is_disk_compressible}; } pub(crate) mod deadlock_detector { #[cfg(test)] - pub(crate) use crate::storage::deadlock_detector::RequestHangDetectionPolicy; - pub(crate) use crate::storage::deadlock_detector::{DeadlockDetector, get_deadlock_detector}; + pub(crate) use crate::storage::storage_api::deadlock_detector_consumer::RequestHangDetectionPolicy; + pub(crate) use crate::storage::storage_api::deadlock_detector_consumer::{DeadlockDetector, get_deadlock_detector}; } pub(crate) mod ecfs { - pub(crate) use crate::storage::ecfs::FS; + pub(crate) use crate::storage::storage_api::FS; } pub(crate) mod error { - pub(crate) use crate::storage::{ + pub(crate) use crate::storage::storage_api::{ DiskError, StorageError, is_all_buckets_not_found, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, }; @@ -674,25 +726,29 @@ pub(crate) mod error { } pub(crate) mod head_prefix { - pub(crate) use crate::storage::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children}; + pub(crate) use crate::storage::storage_api::head_prefix_consumer::{ + head_prefix_not_found_message, probe_prefix_has_children, + }; } pub(crate) mod helper { - pub(crate) use crate::storage::helper::{OperationHelper, spawn_background_with_context}; + pub(crate) use crate::storage::storage_api::helper_consumer::{OperationHelper, spawn_background_with_context}; } pub(crate) mod object_utils { - pub(crate) use crate::storage::to_s3s_etag; + pub(crate) use crate::storage::storage_api::to_s3s_etag; } pub(crate) mod io { #[cfg(test)] - pub(crate) use crate::storage::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader}; - pub(crate) use crate::storage::{DynReader, HashReader, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader}; + pub(crate) use crate::storage::storage_api::{DecryptReader, EncryptReader, HardLimitReader, boxed_reader}; + pub(crate) use crate::storage::storage_api::{ + DynReader, HashReader, WriteEncryption, WritePlan, compression_metadata_value, wrap_reader, + }; } pub(crate) mod options { - pub(crate) use crate::storage::options::{ + pub(crate) use crate::storage::storage_api::options_consumer::{ copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime, extract_metadata_from_mime_with_object_name, filter_object_metadata, get_complete_multipart_upload_opts, get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, parse_copy_source_range, put_opts, @@ -701,47 +757,47 @@ pub(crate) mod options { } pub(crate) mod request_context { - pub(crate) use crate::storage::request_context::{RequestContext, spawn_traced}; + pub(crate) use crate::storage::storage_api::request_context_consumer::{RequestContext, spawn_traced}; } pub(crate) mod sse { - pub(crate) use crate::storage::sse::{ - EncryptionKeyKind, SSEType, build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers, - extract_ssekms_context_from_headers, map_get_object_reader_error, mark_encrypted_multipart_metadata, - }; - pub(crate) use crate::storage::{ + pub(crate) use crate::storage::storage_api::sse_consumer::{ DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, apply_bucket_default_lock_retention, extract_server_side_encryption_from_headers, get_buffer_size_opt_in, sse_decryption, sse_encryption, sse_prepare_encryption, }; + pub(crate) use crate::storage::storage_api::sse_consumer::{ + EncryptionKeyKind, SSEType, build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers, + extract_ssekms_context_from_headers, map_get_object_reader_error, mark_encrypted_multipart_metadata, + }; } pub(crate) mod set_disk { - pub(crate) use crate::storage::{get_lock_acquire_timeout, is_valid_storage_class}; + pub(crate) use crate::storage::storage_api::{get_lock_acquire_timeout, is_valid_storage_class}; } pub(crate) mod storage_class { - pub(crate) use crate::storage::ecstore_config::storageclass::STANDARD; + pub(crate) use crate::storage::storage_api::ecstore_config::storageclass::STANDARD; #[cfg(test)] - pub(crate) use crate::storage::ecstore_config::storageclass::STANDARD_IA; + pub(crate) use crate::storage::storage_api::ecstore_config::storageclass::STANDARD_IA; } pub(crate) mod timeout_wrapper { - pub(crate) use crate::storage::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper}; + pub(crate) use crate::storage::storage_api::timeout_wrapper_consumer::{GetObjectTimeoutPolicy, RequestTimeoutWrapper}; } pub(crate) mod s3_api { pub(crate) mod bucket { - pub(crate) use crate::storage::s3_api::bucket::{ + pub(crate) use crate::storage::storage_api::s3_api_consumer::bucket::{ ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output, build_list_objects_output, build_list_objects_v2_output, parse_list_object_versions_params, parse_list_objects_v2_params, }; - pub(crate) use crate::storage::s3_api::common::rustfs_owner; + pub(crate) use crate::storage::storage_api::s3_api_consumer::common::rustfs_owner; } pub(crate) mod multipart { - pub(crate) use crate::storage::s3_api::multipart::{ + pub(crate) use crate::storage::storage_api::s3_api_consumer::multipart::{ ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output, parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number, }; @@ -749,32 +805,59 @@ pub(crate) mod s3_api { } pub(crate) mod admin_usecase { - pub(crate) use super::storage_contracts::StorageAdminApi; + pub(crate) mod contract { + pub(crate) use super::super::storage_contracts::StorageAdminApi; + } + pub(crate) use super::{admin, capacity, data_usage}; - pub(crate) use crate::storage::{ECStore, EndpointServerPools}; + pub(crate) use crate::storage::storage_api::{ECStore, EndpointServerPools}; } pub(crate) mod bucket_usecase { - pub(crate) use super::storage_contracts::{ - BucketOperations, BucketOptions, DeleteBucketOptions, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, - MakeBucketOptions, - }; + pub(crate) mod contract { + pub(crate) mod bucket { + pub(crate) use super::super::super::storage_contracts::{ + BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, + }; + } + + pub(crate) mod list { + pub(crate) use super::super::super::storage_contracts::{ListObjectVersionsInfo, ListObjectsV2Info, ListOperations}; + } + } + pub(crate) use super::{access, bucket, data_usage, error, helper, object_utils, request_context, s3_api}; - pub(crate) use crate::storage::{ + pub(crate) use crate::storage::storage_api::{ ECStore, StorageObjectInfo, get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations, validate_list_object_unordered_with_delimiter, }; } pub(crate) mod object_usecase { - #[cfg(test)] - pub(crate) use super::storage_contracts::HTTPPreconditions; - pub(crate) use super::storage_contracts::{HTTPRangeSpec, NamespaceLocking, ObjectIO, ObjectOperations}; + pub(crate) mod contract { + #[cfg(test)] + pub(crate) mod http { + pub(crate) use super::super::super::storage_contracts::HTTPPreconditions; + } + + pub(crate) mod namespace { + pub(crate) use super::super::super::storage_contracts::NamespaceLocking; + } + + pub(crate) mod object { + pub(crate) use super::super::super::storage_contracts::{ObjectIO, ObjectOperations}; + } + + pub(crate) mod range { + pub(crate) use super::super::super::storage_contracts::HTTPRangeSpec; + } + } + pub(crate) use super::{ access, bucket, compression, concurrency, data_usage, deadlock_detector, ecfs, error, head_prefix, helper, io, object_utils, options, request_context, s3_api, set_disk, sse, storage_class, timeout_wrapper, }; - pub(crate) use crate::storage::{ + pub(crate) use crate::storage::storage_api::{ ECStore, RFC1123, StorageDeletedObject, StorageObjectInfo, StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, check_preconditions, get_validated_store, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention, parse_part_number_i32_to_usize, remove_object_lock_metadata_for_copy, @@ -784,21 +867,40 @@ pub(crate) mod object_usecase { } pub(crate) mod multipart_usecase { - #[cfg(test)] - pub(crate) use super::storage_contracts::HTTPPreconditions; - pub(crate) use super::storage_contracts::{ - CompletePart, HTTPRangeSpec, MultipartOperations, MultipartUploadResult, ObjectIO, ObjectOperations, - }; + pub(crate) mod contract { + #[cfg(test)] + pub(crate) mod http { + pub(crate) use super::super::super::storage_contracts::HTTPPreconditions; + } + + pub(crate) mod multipart { + pub(crate) use super::super::super::storage_contracts::{CompletePart, MultipartOperations, MultipartUploadResult}; + } + + pub(crate) mod object { + pub(crate) use super::super::super::storage_contracts::{ObjectIO, ObjectOperations}; + } + + pub(crate) mod range { + pub(crate) use super::super::super::storage_contracts::HTTPRangeSpec; + } + } + pub(crate) use super::{ access, bucket, compression, data_usage, error, helper, io, object_utils, options, s3_api, set_disk, sse, }; - pub(crate) use crate::storage::{ECStore, StorageObjectOptions, StoragePutObjReader}; + pub(crate) use crate::storage::storage_api::{ECStore, StorageObjectOptions, StoragePutObjReader}; } pub(crate) mod select_object { - pub(crate) use super::storage_contracts::ObjectOperations; + pub(crate) mod contract { + pub(crate) mod object { + pub(crate) use super::super::super::storage_contracts::ObjectOperations; + } + } + pub(crate) use super::{options, request_context}; - pub(crate) use crate::storage::{get_validated_store, validate_sse_headers_for_read, validate_ssec_for_read}; + pub(crate) use crate::storage::storage_api::{get_validated_store, validate_sse_headers_for_read, validate_ssec_for_read}; } pub(crate) mod context { @@ -806,20 +908,38 @@ pub(crate) mod context { pub(crate) use super::EndpointServerPools; pub(crate) use super::bucket; pub(crate) use super::runtime; - pub(crate) use crate::storage::{ECStore, EndpointServerPools}; + pub(crate) use crate::storage::storage_api::{ECStore, EndpointServerPools}; #[cfg(test)] - pub(crate) use crate::storage::{Endpoint, Endpoints, PoolEndpoints}; + pub(crate) use crate::storage::storage_api::{Endpoint, Endpoints, PoolEndpoints}; } #[cfg(test)] pub(crate) mod test { pub(crate) use super::EndpointServerPools; - pub(crate) use super::storage_contracts::{ - BucketOperations, BucketOptions, HealOperations, ListOperations, MakeBucketOptions, MultipartOperations, ObjectIO, - ObjectOperations, - }; + pub(crate) mod contract { + pub(crate) mod bucket { + pub(crate) use super::super::super::storage_contracts::{BucketOperations, BucketOptions, MakeBucketOptions}; + } + + pub(crate) mod heal { + pub(crate) use super::super::super::storage_contracts::HealOperations; + } + + pub(crate) mod list { + pub(crate) use super::super::super::storage_contracts::ListOperations; + } + + pub(crate) mod multipart { + pub(crate) use super::super::super::storage_contracts::MultipartOperations; + } + + pub(crate) mod object { + pub(crate) use super::super::super::storage_contracts::{ObjectIO, ObjectOperations}; + } + } + pub(crate) use super::{bucket, ecfs, object_utils, runtime}; - pub(crate) use crate::storage::{ + pub(crate) use crate::storage::storage_api::{ ECStore, Endpoint, Endpoints, PoolEndpoints, StorageObjectInfo, StorageObjectOptions, StoragePutObjReader, }; } diff --git a/rustfs/src/capacity/service.rs b/rustfs/src/capacity/service.rs index 73fb8deac..d008d42bd 100644 --- a/rustfs/src/capacity/service.rs +++ b/rustfs/src/capacity/service.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage_api::capacity::{all_local_disk, disk_drive_path, disk_endpoint}; +use crate::storage_api::capacity::service::{all_local_disk, disk_drive_path, disk_endpoint}; use rustfs_io_metrics::capacity_metrics::{ record_capacity_cache_hit, record_capacity_cache_miss, record_capacity_cache_served, record_capacity_refresh_request, record_capacity_scan_mode, diff --git a/rustfs/src/cluster_snapshot.rs b/rustfs/src/cluster_snapshot.rs index 4f9624197..4314508e6 100644 --- a/rustfs/src/cluster_snapshot.rs +++ b/rustfs/src/cluster_snapshot.rs @@ -16,11 +16,13 @@ use crate::runtime_capabilities::runtime_observability_snapshot; use crate::server::{ DependencyReadiness, DependencyReadinessReport, ReadinessDegradedReason, snapshot_dependency_readiness_report, }; +use crate::storage_api::cluster::EndpointServerPools; +use crate::storage_api::cluster::contract::observability::ObservabilitySnapshot; +use crate::storage_api::cluster::contract::topology::TopologySnapshot; use crate::storage_api::cluster::control_plane::{ ClusterControlPlane, ClusterControlPlaneSnapshot, ClusterLocalNodeStorageSnapshot, ClusterMembershipSnapshot, ClusterPeerHealthSnapshot, ClusterPoolStateSnapshot, }; -use crate::storage_api::cluster::{EndpointServerPools, ObservabilitySnapshot, TopologySnapshot}; use crate::workload_admission::workload_admission_registry_snapshot; use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot}; @@ -115,9 +117,9 @@ pub fn cluster_has_actionable_pressure(snapshot: &ClusterReadOnlySnapshot) -> bo #[cfg(test)] mod tests { use super::*; - use crate::storage_api::cluster::{ - CapabilityState, CapabilityStatus, DiskCapabilities, Endpoint, Endpoints, PoolEndpoints, TopologyCapabilities, - }; + use crate::storage_api::cluster::contract::capability::{CapabilityState, CapabilityStatus}; + use crate::storage_api::cluster::contract::topology::{DiskCapabilities, TopologyCapabilities}; + use crate::storage_api::cluster::{Endpoint, Endpoints, PoolEndpoints}; use rustfs_concurrency::{WorkloadAdmissionSnapshot, WorkloadClass}; #[test] diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index 3310ef78e..46b4b9ef0 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -12,7 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage_api::error::{HTTPRangeError, QuotaError, StorageError}; +use crate::storage_api::error::contract::range::HTTPRangeError; +use crate::storage_api::error::{QuotaError, StorageError}; use s3s::{S3Error, S3ErrorCode}; #[derive(Debug)] diff --git a/rustfs/src/init.rs b/rustfs/src/init.rs index e0d0e6a26..2909bee23 100644 --- a/rustfs/src/init.rs +++ b/rustfs/src/init.rs @@ -14,7 +14,7 @@ use crate::runtime_sources::{resolve_notify_interface, resolve_region}; use crate::server::ShutdownHandle; -use crate::storage_api::startup::{ +use crate::storage_api::startup::init::{ get_bucket_notification_config, process_lambda_configurations, process_queue_configurations, process_topic_configurations, }; use crate::{admin, config, startup_runtime_sources, version}; @@ -717,7 +717,7 @@ where /// When enabled, it spawns a background task that tunes concurrency settings /// every 60 seconds. pub async fn init_auto_tuner(ctx: tokio_util::sync::CancellationToken) { - use crate::storage_api::startup::concurrency::get_concurrency_manager; + use crate::storage_api::startup::init::concurrency::get_concurrency_manager; use rustfs_io_metrics::AutoTuner; use rustfs_io_metrics::TunerConfig; use tracing::{debug, error, info}; @@ -850,7 +850,7 @@ pub async fn init_ftp_system() -> Result, Box = FtpsServer::new(config, storage_client).await?; let bind_addr = server.config().bind_addr; @@ -970,7 +970,7 @@ pub async fn init_ftps_system() -> Result, Box = FtpsServer::new(config, storage_client).await?; let bind_addr = server.config().bind_addr; @@ -1088,7 +1088,7 @@ pub async fn init_webdav_system() -> Result, Box = WebDavServer::new(config, storage_client).await?; let bind_addr = server.config().bind_addr; @@ -1222,7 +1222,7 @@ pub async fn init_sftp_system() -> Result, Box CapabilityStatus { #[cfg(test)] mod tests { use super::*; - use crate::storage_api::cluster::{ - CapabilityState, Endpoint, Endpoints, ObservabilitySnapshotProvider, PoolEndpoints, TopologySnapshotProvider, - }; + use crate::storage_api::cluster::contract::capability::CapabilityState; + use crate::storage_api::cluster::contract::observability::ObservabilitySnapshotProvider; + use crate::storage_api::cluster::contract::topology::TopologySnapshotProvider; + use crate::storage_api::cluster::{Endpoint, Endpoints, PoolEndpoints}; #[tokio::test] async fn observability_provider_returns_platform_snapshot() { diff --git a/rustfs/src/server/event.rs b/rustfs/src/server/event.rs index b9b54e3fe..0a1f367eb 100644 --- a/rustfs/src/server/event.rs +++ b/rustfs/src/server/event.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store, runtime_sources}; -use crate::storage_api::server::{EventArgs as EcstoreEventArgs, StorageObjectInfo, register_event_dispatch_hook}; +use crate::storage_api::server::event::{EventArgs as EcstoreEventArgs, StorageObjectInfo, register_event_dispatch_hook}; use chrono::{DateTime, Utc}; use rustfs_notify::{EventArgs as NotifyEventArgs, NotifyObjectInfo}; use rustfs_s3_types::EventName; @@ -234,7 +234,8 @@ pub async fn init_event_notifier() { #[cfg(test)] mod tests { use super::{convert_ecstore_object_info, parse_host_and_port}; - use crate::storage_api::server::{StorageObjectInfo, TransitionedObject}; + use crate::storage_api::server::event::StorageObjectInfo; + use crate::storage_api::server::event::contract::lifecycle::TransitionedObject; use chrono::{DateTime, Utc}; use std::{collections::HashMap, sync::Arc}; use time::{Duration, OffsetDateTime}; diff --git a/rustfs/src/server/http.rs b/rustfs/src/server/http.rs index 5028f9dda..80939f185 100644 --- a/rustfs/src/server/http.rs +++ b/rustfs/src/server/http.rs @@ -30,11 +30,11 @@ use crate::server::{ TlsAcceptorHolder, TlsHandshakeFailureKind, build_acceptor_from_loaded, load_tls_material, spawn_reload_loop, }, }; -use crate::storage_api::server as storage; -use crate::storage_api::server::request_context::{RequestContext, extract_request_id_from_headers}; -use crate::storage_api::server::rpc::InternodeRpcService; -use crate::storage_api::server::tonic_service::make_server; -use crate::storage_api::server::{TONIC_RPC_PREFIX, verify_rpc_signature}; +use crate::storage_api::server::http as storage; +use crate::storage_api::server::http::request_context::{RequestContext, extract_request_id_from_headers}; +use crate::storage_api::server::http::rpc::InternodeRpcService; +use crate::storage_api::server::http::tonic_service::make_server; +use crate::storage_api::server::http::{TONIC_RPC_PREFIX, verify_rpc_signature}; use bytes::Bytes; use http::{HeaderMap, Method, Request as HttpRequest, Response}; use hyper::body::Incoming; @@ -1104,7 +1104,8 @@ fn process_connection( .layer( TraceLayer::new_for_http() .make_span_with(|request: &HttpRequest<_>| { - let request_context = request.extensions().get::(); + let request_context = + request.extensions().get::(); let request_id = request_context .map(|ctx| ctx.request_id.as_str()) .unwrap_or("unknown"); @@ -1253,7 +1254,8 @@ fn process_connection( .layer( TraceLayer::new_for_http() .make_span_with(|request: &HttpRequest<_>| { - let request_context = request.extensions().get::(); + let request_context = + request.extensions().get::(); let request_id = request_context .map(|ctx| ctx.request_id.as_str()) .unwrap_or("unknown"); diff --git a/rustfs/src/server/layer.rs b/rustfs/src/server/layer.rs index cc7a94785..a5573d1a3 100644 --- a/rustfs/src/server/layer.rs +++ b/rustfs/src/server/layer.rs @@ -25,8 +25,8 @@ use crate::server::{ MINIO_HEALTH_READY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX, active_http_requests, has_path_prefix, is_admin_path, is_table_catalog_path, }; -use crate::storage_api::server::apply_cors_headers; -use crate::storage_api::server::request_context::{ +use crate::storage_api::server::layer::apply_cors_headers; +use crate::storage_api::server::layer::request_context::{ RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced, }; use bytes::Bytes; diff --git a/rustfs/src/server/module_switch.rs b/rustfs/src/server/module_switch.rs index d5fccca39..d3af58428 100644 --- a/rustfs/src/server/module_switch.rs +++ b/rustfs/src/server/module_switch.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::runtime_sources; -use crate::storage_api::server::{Error as StorageError, read_config, save_config}; +use crate::storage_api::server::module_switch::{Error as StorageError, read_config, save_config}; use serde::{Deserialize, Serialize}; use std::sync::atomic::{AtomicBool, Ordering}; diff --git a/rustfs/src/server/readiness.rs b/rustfs/src/server/readiness.rs index 4f894a901..48987490a 100644 --- a/rustfs/src/server/readiness.rs +++ b/rustfs/src/server/readiness.rs @@ -15,9 +15,10 @@ use crate::server::runtime_sources; use crate::server::{ServiceState, ServiceStateManager}; use crate::server::{has_path_prefix, is_table_catalog_path}; -use crate::storage_api::server::{Endpoint, EndpointServerPools, StorageAdminApi, is_dist_erasure}; +use crate::storage_api::server::readiness::contract::admin::StorageAdminApi; +use crate::storage_api::server::readiness::{Endpoint, EndpointServerPools, is_dist_erasure}; #[cfg(test)] -use crate::storage_api::server::{Endpoints, PoolEndpoints}; +use crate::storage_api::server::readiness::{Endpoints, PoolEndpoints}; use bytes::Bytes; use http::{Request as HttpRequest, Response, StatusCode}; use http_body::Body; diff --git a/rustfs/src/server/runtime_sources.rs b/rustfs/src/server/runtime_sources.rs index 651e44e1b..bf26d598d 100644 --- a/rustfs/src/server/runtime_sources.rs +++ b/rustfs/src/server/runtime_sources.rs @@ -16,7 +16,7 @@ use crate::runtime_sources::{ NotifyInterface, resolve_endpoints_handle, resolve_iam_ready, resolve_kms_runtime_service_manager, resolve_lock_clients_handle, resolve_notify_interface, resolve_object_store_handle, resolve_server_config, }; -use crate::storage_api::server::{ECStore, EndpointServerPools}; +use crate::storage_api::server::runtime_sources::{ECStore, EndpointServerPools}; use rustfs_kms::KmsServiceManager; use rustfs_lock::LockClient; use std::collections::HashMap; diff --git a/rustfs/src/startup_background.rs b/rustfs/src/startup_background.rs index f28ec9a80..3ba720d59 100644 --- a/rustfs/src/startup_background.rs +++ b/rustfs/src/startup_background.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage_api::startup::{ECStore, set_workload_admission_snapshot_provider}; +use crate::storage_api::startup::background::{ECStore, set_workload_admission_snapshot_provider}; use crate::workload_admission::RustFsWorkloadAdmissionSnapshotProvider; use rustfs_concurrency::WorkloadAdmissionSnapshotProvider; use rustfs_heal::{ diff --git a/rustfs/src/startup_bucket_metadata.rs b/rustfs/src/startup_bucket_metadata.rs index b17a68314..4150e0288 100644 --- a/rustfs/src/startup_bucket_metadata.rs +++ b/rustfs/src/startup_bucket_metadata.rs @@ -13,8 +13,9 @@ // limitations under the License. use crate::startup_runtime_sources; -use crate::storage_api::startup::{ - BucketOperations, BucketOptions, ECStore, init_bucket_metadata_sys, try_migrate_bucket_metadata, try_migrate_iam_config, +use crate::storage_api::startup::bucket_metadata::contract::bucket::{BucketOperations, BucketOptions}; +use crate::storage_api::startup::bucket_metadata::{ + ECStore, init_bucket_metadata_sys, try_migrate_bucket_metadata, try_migrate_iam_config, }; use std::{ io::{Error, Result}, diff --git a/rustfs/src/startup_deadlock.rs b/rustfs/src/startup_deadlock.rs index 2ba2820cc..7a6978b78 100644 --- a/rustfs/src/startup_deadlock.rs +++ b/rustfs/src/startup_deadlock.rs @@ -19,7 +19,7 @@ const LOG_SUBSYSTEM_STARTUP: &str = "startup"; const EVENT_DEADLOCK_DETECTOR_STATE: &str = "deadlock_detector_state"; pub(crate) fn init_deadlock_detector_runtime() { - let detector = crate::storage_api::startup::deadlock_detector::get_deadlock_detector(); + let detector = crate::storage_api::startup::deadlock::get_deadlock_detector(); if detector.is_enabled() { detector.start(); info!( diff --git a/rustfs/src/startup_fs_guard.rs b/rustfs/src/startup_fs_guard.rs index 4440d7efc..53964f2e2 100644 --- a/rustfs/src/startup_fs_guard.rs +++ b/rustfs/src/startup_fs_guard.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage_api::startup::EndpointServerPools; +use crate::storage_api::startup::fs_guard::EndpointServerPools; use rustfs_config::{ DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, ENV_RUSTFS_UNSUPPORTED_FS_POLICY, RUSTFS_UNSUPPORTED_FS_POLICY_FAIL, RUSTFS_UNSUPPORTED_FS_POLICY_WARN, diff --git a/rustfs/src/startup_iam.rs b/rustfs/src/startup_iam.rs index 4ea5e1ffb..e213a582d 100644 --- a/rustfs/src/startup_iam.rs +++ b/rustfs/src/startup_iam.rs @@ -14,7 +14,7 @@ use crate::runtime_sources::AppContext; use crate::server::{ServiceStateManager, publish_ready_when_runtime_ready}; -use crate::storage_api::startup::ECStore; +use crate::storage_api::startup::iam::ECStore; use rustfs_common::{GlobalReadiness, SystemStage}; use rustfs_iam::init_iam_sys; use rustfs_kms::KmsServiceManager; diff --git a/rustfs/src/startup_lifecycle.rs b/rustfs/src/startup_lifecycle.rs index b23746ab6..ecc45340d 100644 --- a/rustfs/src/startup_lifecycle.rs +++ b/rustfs/src/startup_lifecycle.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage_api::startup::ECStore; +use crate::storage_api::startup::lifecycle::ECStore; use crate::{ server::{ServiceStateManager, ShutdownHandle, wait_for_shutdown}, startup_iam::{IamBootstrapDisposition, publish_ready_for_iam_bootstrap}, diff --git a/rustfs/src/startup_notification.rs b/rustfs/src/startup_notification.rs index 143e596e9..738925f4a 100644 --- a/rustfs/src/startup_notification.rs +++ b/rustfs/src/startup_notification.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::init::add_bucket_notification_configuration; -use crate::storage_api::startup::{EndpointServerPools, Result as StorageResult, new_global_notification_sys}; +use crate::storage_api::startup::notification::{EndpointServerPools, Result as StorageResult, new_global_notification_sys}; use std::{ future::Future, io::{Error, Result}, @@ -76,7 +76,7 @@ fn log_embedded_optional_service_skipped(service: &str, err: impl std::fmt::Disp #[cfg(test)] mod tests { use super::init_notification_system_with; - use crate::storage_api::startup::Error as EcstoreError; + use crate::storage_api::startup::notification::Error as EcstoreError; #[tokio::test] async fn notification_system_returns_source_error() { diff --git a/rustfs/src/startup_runtime_sources.rs b/rustfs/src/startup_runtime_sources.rs index 5fb7d32a6..08f5326db 100644 --- a/rustfs/src/startup_runtime_sources.rs +++ b/rustfs/src/startup_runtime_sources.rs @@ -14,7 +14,7 @@ use crate::config::RustFSBufferConfig; use crate::runtime_sources::{resolve_outbound_tls_generation, resolve_replication_pool_handle}; -use crate::storage_api::startup::{DynReplicationPool, set_global_region, set_global_rustfs_port}; +use crate::storage_api::startup::runtime_sources::{DynReplicationPool, set_global_region, set_global_rustfs_port}; use rustfs_kms::KmsServiceManager; use rustfs_obs::{GlobalError as ObservabilityError, OtelGuard}; use rustfs_tls_runtime::{OutboundTlsMaterial, TlsGeneration}; diff --git a/rustfs/src/startup_services.rs b/rustfs/src/startup_services.rs index c879b9715..0aa28d1f6 100644 --- a/rustfs/src/startup_services.rs +++ b/rustfs/src/startup_services.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage_api::startup::{ECStore, EndpointServerPools}; +use crate::storage_api::startup::services::{ECStore, EndpointServerPools}; use crate::{ config::Config, init::{init_buffer_profile_system, init_kms_system}, diff --git a/rustfs/src/startup_shutdown.rs b/rustfs/src/startup_shutdown.rs index d8466b019..3fccdc283 100644 --- a/rustfs/src/startup_shutdown.rs +++ b/rustfs/src/startup_shutdown.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage_api::startup::shutdown_background_services; +use crate::storage_api::startup::shutdown::shutdown_background_services; use crate::{ server::{ServiceState, ServiceStateManager, ShutdownHandle, ShutdownSignal, shutdown_event_notifier, stop_audit_system}, startup_optional_runtime_sidecars::{ diff --git a/rustfs/src/startup_storage.rs b/rustfs/src/startup_storage.rs index 8c3f11775..27e285b51 100644 --- a/rustfs/src/startup_storage.rs +++ b/rustfs/src/startup_storage.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::startup_fs_guard::enforce_unsupported_fs_policy; -use crate::storage_api::startup::{ +use crate::storage_api::startup::storage::{ ECStore, EndpointServerPools, init_background_replication, init_ecstore_config, init_global_config_sys, init_local_disks, init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints, try_migrate_server_config, update_erasure_type, }; diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index bdb8f05af..fc3bec0d6 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -22,9 +22,9 @@ use crate::auth::{check_key_valid, get_condition_values_with_query_and_client_in use crate::error::ApiError; use crate::license::license_check; use crate::server::RemoteAddr; -use crate::storage::contract::bucket::BucketOperations; use crate::storage::request_context::RequestContext; -use crate::storage::runtime_sources; +use crate::storage::storage_api::access_consumer::contract::bucket::BucketOperations; +use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; use metrics::counter; use rustfs_iam::error::Error as IamError; use rustfs_policy::policy::action::{Action, AdminAction, S3Action}; @@ -2051,10 +2051,23 @@ impl S3Access for FS { } #[cfg(test)] +#[allow(unused_imports)] mod tests { - use super::*; + use super::{ + AMZ_WRITE_OFFSET_BYTES_HEADER, BucketPolicyArgs, BucketPolicyExistingObjectTagHint, BucketPolicyRawLoadErrorKind, FS, + ObjectTagConditions, PostObjectRequestMarker, ReqInfo, S3Access, StorageError, + bucket_policy_needs_existing_object_tag_from_hint, classify_bucket_policy_raw_load_error, + complete_multipart_upload_authorize_action, get_bucket_policy_authorize_action, has_write_offset_bytes_header, + legal_hold_write_requested, list_parts_authorize_action, load_bucket_policy_existing_object_tag_hint, + merge_list_bucket_query_conditions, owner_can_bypass_policy_deny, post_object_authorize_action, + put_bucket_policy_authorize_action, request_context_from_req, retention_write_requested, secondary_tag_hint_action, + table_data_plane_admin_action, validate_post_object_success_controls, + }; + use crate::error::ApiError; use http::{Extensions, HeaderMap, Method, Uri}; + use rustfs_policy::policy::action::{Action, S3Action}; use rustfs_policy::policy::{BucketPolicy, bucket_policy_uses_existing_object_tag_conditions}; + use s3s::{S3ErrorCode, S3Request, dto::*}; use std::collections::HashMap; use time::OffsetDateTime; diff --git a/rustfs/src/storage/backpressure.rs b/rustfs/src/storage/backpressure.rs index 76eff2563..9585ba8a8 100644 --- a/rustfs/src/storage/backpressure.rs +++ b/rustfs/src/storage/backpressure.rs @@ -483,8 +483,9 @@ impl Default for BackpressureMonitor { } #[cfg(test)] +#[allow(unused_imports)] mod tests { - use super::*; + use super::{BackpressureMonitor, BackpressurePipe, BackpressureState, ObjectPipeBackpressurePolicy}; #[test] fn test_backpressure_config_default() { diff --git a/rustfs/src/storage/concurrency/io_schedule.rs b/rustfs/src/storage/concurrency/io_schedule.rs index afaf59ce3..d611dfeb2 100644 --- a/rustfs/src/storage/concurrency/io_schedule.rs +++ b/rustfs/src/storage/concurrency/io_schedule.rs @@ -1930,9 +1930,17 @@ pub fn get_buffer_size_opt_in(file_size: i64) -> usize { // ============================================ #[cfg(test)] +#[allow(unused_imports)] mod tests { - use super::*; + use super::{ + IoLoadLevel, IoPriority, IoPriorityMetrics, IoPriorityQueue, IoPriorityQueueConfig, IoSchedulerConfig, + IoSchedulingContext, IoStrategy, get_advanced_buffer_size, get_buffer_size_opt_in, get_concurrency_aware_buffer_size, + }; + use rustfs_io_core::io_profile::{AccessPattern, StorageMedia}; + use rustfs_io_metrics::bandwidth::{BandwidthSnapshot, BandwidthTier}; use serial_test::serial; + use std::sync::atomic::Ordering; + use std::time::Duration; #[tokio::test] #[serial] diff --git a/rustfs/src/storage/concurrency/manager.rs b/rustfs/src/storage/concurrency/manager.rs index 5eed85c5b..a577178b2 100644 --- a/rustfs/src/storage/concurrency/manager.rs +++ b/rustfs/src/storage/concurrency/manager.rs @@ -19,7 +19,7 @@ use super::io_schedule::{ get_advanced_buffer_size, }; use super::request_guard::{GetObjectGuard, PutObjectGuard}; -use crate::storage::runtime_sources; +use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; use rustfs_concurrency::{ AdmissionState, GetObjectQueueSnapshot, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider, WorkloadClass, @@ -664,9 +664,16 @@ impl Default for ConcurrencyManager { // ============================================ #[cfg(test)] +#[allow(unused_imports)] mod integration_tests { - use super::*; + use super::super::io_schedule::{IoLoadLevel, IoPriority}; + use super::super::request_guard::GetObjectGuard; + use super::ConcurrencyManager; + use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshotProvider, WorkloadClass}; + use rustfs_config::MI_B; + use rustfs_io_core::io_profile::{AccessPattern, StorageMedia}; use serial_test::serial; + use std::time::Duration; #[tokio::test] #[serial] diff --git a/rustfs/src/storage/concurrency/request_guard.rs b/rustfs/src/storage/concurrency/request_guard.rs index 72bff1519..7737192c9 100644 --- a/rustfs/src/storage/concurrency/request_guard.rs +++ b/rustfs/src/storage/concurrency/request_guard.rs @@ -173,7 +173,7 @@ impl Drop for PutObjectGuard { #[cfg(test)] mod tests { - use super::*; + use super::{GetObjectGuard, PutObjectGuard}; #[test] fn test_guard_increments_counter() { diff --git a/rustfs/src/storage/deadlock_detector.rs b/rustfs/src/storage/deadlock_detector.rs index 108b9718b..8783660d1 100644 --- a/rustfs/src/storage/deadlock_detector.rs +++ b/rustfs/src/storage/deadlock_detector.rs @@ -611,8 +611,10 @@ pub fn is_deadlock_detection_enabled() -> bool { } #[cfg(test)] +#[allow(unused_imports)] mod tests { - use super::*; + use super::{DeadlockDetector, LockInfo, LockType, RequestHangDetectionPolicy, RequestResourceTracker, WaitGraphEdge}; + use std::time::{Duration, Instant}; #[test] fn test_deadlock_detector_config_default() { diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 75194eec1..86d1503c5 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -23,15 +23,17 @@ use super::{ use super::{StorageReplicationConfigExt as _, StorageVersioningConfigExt as _}; use crate::error::ApiError; use crate::storage::access::has_bypass_governance_header; -use crate::storage::contract::{ +use crate::storage::helper::OperationHelper; +use crate::storage::options::get_opts; +use crate::storage::s3_api::{self, acl}; +use crate::storage::storage_api::ecfs_consumer::contract::{ bucket::{BucketOperations, BucketOptions}, object::{ObjectLockRetentionOptions, ObjectOperations as _}, }; -use crate::storage::helper::OperationHelper; -use crate::storage::options::get_opts; -use crate::storage::runtime_sources; -use crate::storage::s3_api::{self, acl}; -use crate::storage::{parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled}; +use crate::storage::storage_api::ecfs_consumer::object_lock::{ + parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled, +}; +use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; use crate::table_catalog; use http::StatusCode; use metrics::{counter, histogram}; @@ -52,7 +54,7 @@ const LOG_SUBSYSTEM_OBJECT: &str = "object"; const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock"; const LOG_SUBSYSTEM_TAGGING: &str = "tagging"; -use crate::storage::StorageObjectOptions as ObjectOptions; +use crate::storage::storage_api::ecfs_consumer::StorageObjectOptions as ObjectOptions; #[derive(Debug, Clone)] pub struct FS { diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index cd1d40a40..47537eeb6 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -20,11 +20,11 @@ use super::{ use crate::config::{RustFSBufferConfig, WorkloadProfile, is_buffer_profile_enabled}; use crate::error::ApiError; use crate::server::cors; -use crate::storage::contract::{ +use crate::storage::ecfs::ListObjectUnorderedQuery; +use crate::storage::storage_api::ecfs_extend_consumer::contract::{ bucket::{BucketOperations, BucketOptions}, object::ObjectToDelete, }; -use crate::storage::ecfs::ListObjectUnorderedQuery; use http::header::{IF_MATCH, IF_MODIFIED_SINCE, IF_NONE_MATCH, IF_UNMODIFIED_SINCE}; use http::{HeaderMap, HeaderValue, StatusCode}; use metrics::counter; @@ -49,8 +49,8 @@ use time::format_description::well_known::Rfc3339; use time::{format_description::FormatItem, macros::format_description}; use tracing::{debug, warn}; -use crate::storage::StorageObjectInfo as ObjectInfo; -use crate::storage::runtime_sources; +use crate::storage::storage_api::ecfs_extend_consumer::StorageObjectInfo as ObjectInfo; +use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; const LOG_COMPONENT_STORAGE: &str = "storage"; const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock"; diff --git a/rustfs/src/storage/ecfs_test.rs b/rustfs/src/storage/ecfs_test.rs index d558cfc9b..fe784495b 100644 --- a/rustfs/src/storage/ecfs_test.rs +++ b/rustfs/src/storage/ecfs_test.rs @@ -14,16 +14,17 @@ #[cfg(test)] mod tests { - use super::super::{BucketMetadata, DEFAULT_READ_BUFFER_SIZE, get_global_bucket_metadata_sys, set_bucket_metadata}; use crate::config::WorkloadProfile; use crate::server::cors; use crate::storage::ecfs::{FS, validate_object_lock_configuration_input}; use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; - use crate::storage::{ - StorageObjectInfo as ObjectInfo, apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions, - get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal, matches_origin_pattern, parse_etag, - parse_object_lock_legal_hold, parse_object_lock_retention, process_lambda_configurations, process_queue_configurations, - process_topic_configurations, remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata, + use crate::storage::storage_api::test_consumer::{ + BucketMetadata, DEFAULT_READ_BUFFER_SIZE, StorageObjectInfo as ObjectInfo, apply_cors_headers, + apply_default_lock_retention_metadata, bucket_metadata_sys_initialized, check_preconditions, decode_tags_to_map, + get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, get_global_bucket_metadata_sys, is_etag_equal, + matches_origin_pattern, parse_etag, parse_object_lock_legal_hold, parse_object_lock_retention, + process_lambda_configurations, process_queue_configurations, process_topic_configurations, + remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata, set_bucket_metadata, validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter, }; use http::{Extensions, HeaderMap, HeaderValue, Method, StatusCode, Uri}; @@ -953,8 +954,6 @@ mod tests { #[tokio::test] async fn test_validate_bucket_object_lock_enabled() { - use super::super::bucket_metadata_sys_initialized; - use super::super::set_bucket_metadata; use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled}; use time::OffsetDateTime; @@ -1793,7 +1792,6 @@ mod tests { /// with a single-element vec value, matching the format expected by policy evaluation. #[test] fn test_object_tag_condition_key_format() { - use super::super::decode_tags_to_map; use std::collections::HashMap; let tags_str = "security=public&project=webapp&env=prod"; diff --git a/rustfs/src/storage/head_prefix.rs b/rustfs/src/storage/head_prefix.rs index 7c827b33e..49f53f452 100644 --- a/rustfs/src/storage/head_prefix.rs +++ b/rustfs/src/storage/head_prefix.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::ECStore; -use crate::storage::contract::list::ListOperations as _; +use crate::storage::storage_api::head_prefix_consumer::contract::list::ListOperations as _; use std::sync::Arc; /// Determines if the key "looks like a prefix" (ends with `/`). diff --git a/rustfs/src/storage/helper.rs b/rustfs/src/storage/helper.rs index 92f6dacf2..8867001d8 100644 --- a/rustfs/src/storage/helper.rs +++ b/rustfs/src/storage/helper.rs @@ -15,7 +15,7 @@ use crate::server::{convert_ecstore_object_info, is_audit_module_enabled, is_notify_module_enabled}; use crate::storage::access::{ReqInfo, request_context_from_req}; use crate::storage::request_context::{RequestContext, extract_request_id_from_headers}; -use crate::storage::runtime_sources; +use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; use hashbrown::HashMap; use http::StatusCode; use metrics::counter; @@ -37,7 +37,7 @@ use std::future::Future; use tokio::runtime::{Builder, Handle}; use tracing::{Instrument, info_span, warn}; -use crate::storage::StorageObjectInfo as ObjectInfo; +use crate::storage::storage_api::helper_consumer::StorageObjectInfo as ObjectInfo; /// Schedules an asynchronous task on the current runtime; /// if there is no runtime, creates a minimal runtime execution on a new thread. @@ -391,12 +391,18 @@ impl Drop for OperationHelper { } #[cfg(test)] +#[allow(unused_imports)] mod tests { - use super::*; + use super::OperationHelper; use crate::server::{refresh_audit_module_enabled, refresh_notify_module_enabled}; + use crate::storage::access::ReqInfo; + use crate::storage::request_context::RequestContext; use http::{Extensions, HeaderMap, HeaderValue, Method, Uri}; use metrics::{Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Metadata, SharedString, Unit}; use rustfs_credentials::Credentials; + use rustfs_s3_ops::S3Operation; + use rustfs_s3_types::EventName; + use s3s::S3Request; use s3s::dto::DeleteObjectTaggingInput; use std::sync::{Arc, Mutex}; use temp_env::with_vars; diff --git a/rustfs/src/storage/lock_optimizer.rs b/rustfs/src/storage/lock_optimizer.rs index c31b2e0df..bff2356cf 100644 --- a/rustfs/src/storage/lock_optimizer.rs +++ b/rustfs/src/storage/lock_optimizer.rs @@ -381,9 +381,12 @@ pub fn is_lock_optimization_enabled() -> bool { } #[cfg(test)] +#[allow(unused_imports)] mod tests { - use super::*; + use super::{LockOptimizeConfig, LockOptimizer, LockStats, OptimizedLockGuard}; use std::sync::Mutex; + use std::sync::atomic::Ordering; + use std::time::Duration; #[test] fn test_lock_optimize_config_default() { diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 16e1dcfd7..8c19c2528 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -40,12 +40,51 @@ mod multi_factor_scheduler_integration_test; pub(crate) mod runtime_sources; #[cfg(test)] mod sse_test; -mod storage_api; +pub(crate) mod storage_api; pub(crate) use ecfs_extend::*; -pub(crate) use sse::{ - DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, extract_server_side_encryption_from_headers, sse_decryption, - sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, - validate_sse_headers_for_write, validate_ssec_for_read, +#[allow(unused_imports)] +pub(crate) use storage_api::{ + BUCKET_ACCELERATE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, BUCKET_TABLE_CATALOG_META_PREFIX, + BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG, BUCKET_TABLE_RESERVED_PREFIX, BUCKET_VERSIONING_CONFIG, + BUCKET_WEBSITE_CONFIG, BucketBandwidthMonitor, BucketMetadata, BucketVersioningSys, CheckPartsResp, CollectMetricsOpts, + DEFAULT_READ_BUFFER_SIZE, DailyAllTierStats, DeleteOptions, DiskError, DiskInfo, DiskInfoOptions, DiskResult, DiskStore, + DynReader, DynReplicationPool, ECStore, Endpoint, EndpointServerPools, Error, EventArgs, ExpiryState, FS, FileInfoVersions, + FileReader, FileWriter, GetObjectReader, HashReader, LocalPeerS3Client, MetricType, NotificationSys, OBJECT_LOCK_CONFIG, + ObjectInfo, ObjectLockBlockReason, ObjectOptions, ObjectPartInfo, ObjectStoreResolver, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, + PolicySys, PoolEndpoints, PutObjReader, QuotaError, RUSTFS_META_BUCKET, RawFileInfo, ReadMultipleReq, ReadMultipleResp, + ReadOptions, RenameDataResp, ReplicationStats, Result, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, + SetupType, StorageDeletedObject, StorageDiskRpcExt, StorageError, StorageGetObjectReader, StorageObjectInfo, + StorageObjectOptions, StorageObjectToDelete, StoragePeerS3ClientExt, StoragePutObjReader, StorageReplicationConfigExt, + StorageVersioningConfigExt, TONIC_RPC_PREFIX, TierConfigMgr, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, + WorkloadAdmissionSnapshotProviderRef, WriteEncryption, WritePlan, access_consumer, add_object_lock_years, all_local_disk, + all_local_disk_path, bucket_metadata_runtime_initialized, check_retention_for_modification, collect_local_metrics, + compression_metadata_value, contract, decode_tags, decode_tags_to_map, delete_bucket_metadata_config, disk_drive_path, + disk_endpoint, ecfs_consumer, ecfs_extend_consumer, ecstore_admin, ecstore_bucket, ecstore_capacity, ecstore_client, + ecstore_cluster, ecstore_compression, ecstore_config, ecstore_data_usage, ecstore_disk, ecstore_error, ecstore_event, + ecstore_global, ecstore_layout, ecstore_metrics, ecstore_notification, ecstore_rebalance, ecstore_rio, ecstore_rpc, + ecstore_set_disk, ecstore_storage, ecstore_tier, encode_tags, find_local_disk_by_ref, get_bucket_accelerate_config, + get_bucket_cors_config, get_bucket_logging_config, get_bucket_metadata, get_bucket_notification_config, + get_bucket_object_lock_config, get_bucket_policy_raw, get_bucket_replication_config, get_bucket_request_payment_config, + get_bucket_sse_config, get_bucket_website_config, get_daily_all_tier_stats, get_global_boot_time, get_global_bucket_monitor, + get_global_deployment_id, get_global_endpoints_opt, get_global_expiry_state, get_global_lock_client, get_global_lock_clients, + get_global_notification_sys, get_global_region, get_global_replication_pool, get_global_replication_stats, + get_global_tier_config_mgr, get_local_server_property, get_lock_acquire_timeout, get_public_access_block_config, + global_rustfs_port, head_prefix_consumer, helper_consumer, init_background_replication, init_bucket_metadata_sys, + init_ecstore_config, init_global_config_sys, init_local_disks, init_lock_clients, is_all_buckets_not_found, is_dist_erasure, + is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found, is_valid_storage_class, load_bucket_metadata, + new_global_notification_sys, new_object_layer_fn, options_consumer, prewarm_local_disk_id_map, read_config, + record_replication_proxy, register_event_dispatch_hook, reload_transition_tier_config, replication_queue_current_count, + resolve_object_store_handle, rpc_consumer, runtime_sources_consumer, s3_api_consumer, save_config, serialize, + set_bucket_metadata, set_global_endpoints, set_global_region, set_global_rustfs_port, set_object_store_resolver, + set_workload_admission_snapshot_provider, shutdown_background_services, table_catalog_path_hash, to_s3s_etag, + topology_snapshot_from_endpoint_pools_with_capabilities, try_migrate_bucket_metadata, try_migrate_iam_config, + try_migrate_server_config, update_bucket_metadata_config, update_erasure_type, verify_rpc_signature, wrap_reader, +}; + +#[cfg(test)] +#[allow(unused_imports)] +pub(crate) use storage_api::{ + BucketMetadataSys, DecryptReader, DisksLayout, EncryptReader, Endpoints, HardLimitReader, STORAGE_CLASS_SUB_SYS, + boxed_reader, get_global_bucket_metadata_sys, test_consumer, }; -pub(crate) use storage_api::*; diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 1deca5798..337853b48 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -13,7 +13,7 @@ // limitations under the License. use super::{BucketVersioningSys, Result, StorageError}; -use crate::storage::contract::{object::HTTPPreconditions, range::HTTPRangeSpec}; +use crate::storage::storage_api::options_consumer::contract::{object::HTTPPreconditions, range::HTTPRangeSpec}; use http::header::{IF_MATCH, IF_NONE_MATCH}; use http::{HeaderMap, HeaderValue}; use rustfs_utils::http::{ @@ -44,10 +44,7 @@ use crate::auth::AuthType; use crate::auth::get_query_param; use crate::auth::get_request_auth_type_with_query; use crate::auth::is_request_presigned_signature_v4_with_query; -use crate::storage::StorageObjectOptions as ObjectOptions; - -#[cfg(test)] -use rustfs_utils::http::insert_header; +use crate::storage::storage_api::options_consumer::StorageObjectOptions as ObjectOptions; /// Creates options for deleting an object in a bucket. pub async fn del_opts( @@ -782,12 +779,24 @@ fn get_content_sha256_cksum(headers: &HeaderMap, service_type: Serv } #[cfg(test)] +#[allow(unused_imports)] mod tests { use proptest::prelude::*; use temp_env; - use super::*; + use super::super::StorageError; + use super::{ + ENV_REJECT_ARCHIVE_CONTENT_ENCODING, SUPPORTED_HEADERS, copy_dst_opts, copy_src_opts, del_opts, + detect_content_type_from_object_name, extract_metadata, extract_metadata_from_mime, + extract_metadata_from_mime_with_object_name, filter_object_metadata, get_default_opts, get_opts, parse_copy_source_range, + put_opts, put_opts_from_headers, validate_archive_content_encoding, + }; use http::{HeaderMap, HeaderValue}; + use rustfs_utils::http::{ + AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER, + SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_REQUEST, insert_header, + }; + use s3s::S3ErrorCode; use std::collections::HashMap; use uuid::Uuid; diff --git a/rustfs/src/storage/request_context.rs b/rustfs/src/storage/request_context.rs index 8829e1b3e..6f3965140 100644 --- a/rustfs/src/storage/request_context.rs +++ b/rustfs/src/storage/request_context.rs @@ -192,8 +192,11 @@ where } #[cfg(test)] +#[allow(unused_imports)] mod tests { - use super::*; + use super::{RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers}; + use http::HeaderMap; + use opentelemetry::global; use opentelemetry::trace::{SpanContext, TraceContextExt, TraceFlags, TraceId, TraceState, TracerProvider as _}; use opentelemetry_sdk::propagation::TraceContextPropagator; use opentelemetry_sdk::trace::SdkTracerProvider; diff --git a/rustfs/src/storage/rpc/bucket.rs b/rustfs/src/storage/rpc/bucket.rs index 63ea2fb72..d0161c9ca 100644 --- a/rustfs/src/storage/rpc/bucket.rs +++ b/rustfs/src/storage/rpc/bucket.rs @@ -12,7 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::*; +use super::NodeService; +use crate::storage::storage_api::rpc_consumer::node_service::contract::bucket::{ + BucketOptions, DeleteBucketOptions, MakeBucketOptions, +}; +use crate::storage::storage_api::rpc_consumer::node_service::{ + DiskError, StoragePeerS3ClientExt as _, load_bucket_metadata, set_bucket_metadata, +}; +use rustfs_common::heal_channel::HealOpts; +use rustfs_protos::proto_gen::node_service::*; +use tonic::{Request, Response, Status}; +use tracing::debug; impl NodeService { pub(super) async fn handle_delete_bucket_metadata( diff --git a/rustfs/src/storage/rpc/disk.rs b/rustfs/src/storage/rpc/disk.rs index fa39af177..e7dca1444 100644 --- a/rustfs/src/storage/rpc/disk.rs +++ b/rustfs/src/storage/rpc/disk.rs @@ -12,13 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::*; -use crate::storage::runtime_sources; +use super::NodeService; +use crate::storage::storage_api::rpc_consumer::node_service::{ + DeleteOptions, DiskError, DiskInfoOptions, FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, + StorageDiskRpcExt as _, UpdateMetadataOpts, +}; +use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; +use bytes::Bytes; +use rustfs_filemeta::FileInfo; use rustfs_io_metrics::internode_metrics::{ INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC, }; +use rustfs_protos::proto_gen::node_service::*; use serde::de::DeserializeOwned; use std::io::Cursor; +use tonic::{Request, Response, Status}; +use tracing::debug; fn decode_msgpack_or_json(binary: &[u8], json: &str, value_name: &str) -> std::result::Result { if !binary.is_empty() { @@ -1036,7 +1045,7 @@ impl NodeService { #[cfg(test)] mod tests { - use super::*; + use super::{decode_msgpack_or_json, encode_msgpack}; use serde::{Deserialize, Serialize}; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/rustfs/src/storage/rpc/event.rs b/rustfs/src/storage/rpc/event.rs index 7a574e0e3..2380c7cc6 100644 --- a/rustfs/src/storage/rpc/event.rs +++ b/rustfs/src/storage/rpc/event.rs @@ -12,8 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::*; +use super::NodeService; +use bytes::Bytes; use rustfs_notify::notification_system; +use rustfs_protos::proto_gen::node_service::*; +use tonic::{Request, Response, Status}; impl NodeService { pub(super) async fn handle_get_live_events( diff --git a/rustfs/src/storage/rpc/health.rs b/rustfs/src/storage/rpc/health.rs index 16cc1de7a..f82a2c226 100644 --- a/rustfs/src/storage/rpc/health.rs +++ b/rustfs/src/storage/rpc/health.rs @@ -12,9 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::*; +use super::NodeService; use crate::storage::rpc::encode_msgpack_map; -use crate::storage::runtime_sources; +use crate::storage::storage_api::rpc_consumer::node_service::contract::admin::StorageAdminApi; +use crate::storage::storage_api::rpc_consumer::node_service::get_local_server_property; +use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; +use bytes::Bytes; +use rustfs_madmin::health::{ + get_cpus, get_mem_info, get_os_info, get_partitions, get_proc_info, get_sys_config, get_sys_errors, get_sys_services, +}; +use rustfs_madmin::net::get_net_info; +use rustfs_protos::proto_gen::node_service::*; +use tonic::{Request, Response, Status}; impl NodeService { pub(super) async fn handle_get_proc_info( @@ -240,7 +249,10 @@ impl NodeService { #[cfg(test)] mod tests { - use super::*; + use crate::storage::rpc::encode_msgpack_map; + use rmp_serde::Deserializer; + use serde::Deserialize; + use std::io::Cursor; #[test] fn local_storage_info_rpc_payload_uses_msgpack_map_encoding() { diff --git a/rustfs/src/storage/rpc/http_service.rs b/rustfs/src/storage/rpc/http_service.rs index 247a19879..f8bcf7ff1 100644 --- a/rustfs/src/storage/rpc/http_service.rs +++ b/rustfs/src/storage/rpc/http_service.rs @@ -12,14 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::DEFAULT_READ_BUFFER_SIZE; -use super::super::StorageDiskRpcExt as _; -use super::super::WalkDirOptions; -use super::super::find_local_disk_by_ref; -use super::super::verify_rpc_signature; use crate::server::RPC_PREFIX; use crate::storage::request_context::spawn_traced; -use crate::storage::runtime_sources; +use crate::storage::storage_api::rpc_consumer::http_service::{ + DEFAULT_READ_BUFFER_SIZE, StorageDiskRpcExt as _, WalkDirOptions, find_local_disk_by_ref, verify_rpc_signature, +}; +use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; use bytes::{Bytes, BytesMut}; use futures_util::TryStreamExt; use http::{HeaderMap, Method, Request, Response, StatusCode, Uri}; @@ -716,7 +714,17 @@ fn put_file_stage_error_message(stage: &str, query: &PutFileQuery, err: &dyn std #[cfg(test)] mod tests { - use super::*; + use super::{ + LOG_SUBSYSTEM_DIRECTORY_WALK, LOG_SUBSYSTEM_FILE_TRANSFER, LOG_SUBSYSTEM_ROUTING, PUT_FILE_STREAM_PATH, PutFileQuery, + READ_FILE_STREAM_PATH, WALK_DIR_PATH, internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, + put_file_stage_error_message, read_file_body_stream, verify_internode_rpc_signature, write_body_chunks_to_writer, + }; + use bytes::Bytes; + use http::{HeaderMap, Method, StatusCode, Uri}; + use rustfs_io_metrics::internode_metrics::{ + INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_OPERATION_WALK_DIR, + }; + use tokio::io; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio_stream::StreamExt; use tokio_stream::iter; diff --git a/rustfs/src/storage/rpc/lock.rs b/rustfs/src/storage/rpc/lock.rs index ac6986a61..ff53e706e 100644 --- a/rustfs/src/storage/rpc/lock.rs +++ b/rustfs/src/storage/rpc/lock.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::*; +use super::NodeService; +use rustfs_lock::LockRequest; +use rustfs_protos::proto_gen::node_service::*; +use tonic::{Request, Response, Status}; fn lock_result_from_response(response: rustfs_lock::LockResponse) -> GenerallyLockResult { GenerallyLockResult { @@ -269,7 +272,7 @@ impl NodeService { #[cfg(test)] mod tests { - use super::*; + use super::{lock_result_from_release, lock_result_from_response}; fn test_lock_id() -> rustfs_lock::LockId { rustfs_lock::LockRequest::new(rustfs_lock::ObjectKey::new("bucket", "object"), rustfs_lock::LockType::Exclusive, "owner") diff --git a/rustfs/src/storage/rpc/metrics.rs b/rustfs/src/storage/rpc/metrics.rs index 9a4a58418..79e78e076 100644 --- a/rustfs/src/storage/rpc/metrics.rs +++ b/rustfs/src/storage/rpc/metrics.rs @@ -12,8 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::*; +use super::NodeService; use crate::storage::rpc::encode_msgpack_map; +use crate::storage::storage_api::rpc_consumer::node_service::{CollectMetricsOpts, MetricType, collect_local_metrics}; +use bytes::Bytes; +use rmp_serde::Deserializer; +use rustfs_protos::proto_gen::node_service::*; +use serde::Deserialize; +use std::io::Cursor; +use tonic::{Request, Response, Status}; +use tracing::error; impl NodeService { pub(super) async fn handle_get_metrics( diff --git a/rustfs/src/storage/rpc/mod.rs b/rustfs/src/storage/rpc/mod.rs index 86c5f8f90..38c249061 100644 --- a/rustfs/src/storage/rpc/mod.rs +++ b/rustfs/src/storage/rpc/mod.rs @@ -34,10 +34,10 @@ pub(crate) fn encode_msgpack_map(value: &T) -> Result, rmp #[cfg(test)] mod tests { - use super::*; - use rmp_serde::Deserializer; + use super::encode_msgpack_map; + use rmp_serde::{Deserializer, Serializer}; use rustfs_madmin::{BackendDisks, BackendInfo, Disk, ITEM_ONLINE, StorageInfo}; - use serde::Deserialize; + use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::io::Cursor; diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index b830a1d93..f35f2d277 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -12,34 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::super::{ - CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, ECStore, Error, FileInfoVersions, - LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions, - SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StoragePeerS3ClientExt as _, - UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, get_local_server_property, - load_bucket_metadata, reload_transition_tier_config, resolve_object_store_handle, set_bucket_metadata, -}; use crate::admin::service::{ config::{reload_dynamic_config_runtime_state, reload_runtime_config_snapshot}, site_replication::reload_site_replication_runtime_state, }; -use crate::storage::contract::{ - admin::StorageAdminApi, - bucket::{BucketOptions, DeleteBucketOptions, MakeBucketOptions}, +#[cfg(test)] +use crate::storage::storage_api::rpc_consumer::node_service::STORAGE_CLASS_SUB_SYS; +#[cfg(test)] +use crate::storage::storage_api::rpc_consumer::node_service::{CollectMetricsOpts, MetricType}; +use crate::storage::storage_api::rpc_consumer::node_service::{ + DiskStore, ECStore, Error, LocalPeerS3Client, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SERVICE_SIGNAL_REFRESH_CONFIG, + SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt as _, StorageResult, all_local_disk_path, find_local_disk_by_ref, + reload_transition_tier_config, resolve_object_store_handle, }; -use crate::storage::runtime_sources; +use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; use bytes::Bytes; use futures::Stream; use futures_util::future::join_all; use rmp_serde::Deserializer; -use rustfs_common::heal_channel::HealOpts; -use rustfs_filemeta::{FileInfo, MetacacheReader}; +use rustfs_filemeta::MetacacheReader; use rustfs_iam::store::UserType; -use rustfs_lock::{LockClient, LockRequest}; -use rustfs_madmin::health::{ - get_cpus, get_mem_info, get_os_info, get_partitions, get_proc_info, get_sys_config, get_sys_errors, get_sys_services, -}; -use rustfs_madmin::net::get_net_info; +use rustfs_lock::LockClient; use rustfs_protos::{ models::{PingBody, PingBodyBuilder}, proto_gen::node_service::{node_service_server::NodeService as Node, *}, @@ -127,11 +120,11 @@ fn unimplemented_rpc(method: &str) -> Status { Status::unimplemented(format!("{method} is not implemented")) } -fn background_rebalance_start_error_message(result: super::super::Result<()>) -> Option { +fn background_rebalance_start_error_message(result: StorageResult<()>) -> Option { result.err().map(|err| format!("start_rebalance failed: {err}")) } -fn stop_rebalance_response(result: super::super::Result<()>) -> StopRebalanceResponse { +fn stop_rebalance_response(result: StorageResult<()>) -> StopRebalanceResponse { match result { Ok(_) => StopRebalanceResponse { success: true, @@ -144,7 +137,7 @@ fn stop_rebalance_response(result: super::super::Result<()>) -> StopRebalanceRes } } -fn ensure_rpc_decommission_local_leader(store: &ECStore, idx: usize) -> super::super::Result<()> { +fn ensure_rpc_decommission_local_leader(store: &ECStore, idx: usize) -> StorageResult<()> { let endpoints = store.endpoints(); let endpoint = endpoints .as_ref() @@ -1219,26 +1212,33 @@ impl Node for NodeService { #[cfg(test)] #[allow(unused_imports)] mod tests { - use super::*; - use Request; + use super::{ + CollectMetricsOpts, Error, MetricType, Node as _, NodeService, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, + SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, STORAGE_CLASS_SUB_SYS, + background_rebalance_start_error_message, make_server, stop_rebalance_response, + }; + use bytes::Bytes; + use rustfs_protos::models::PingBodyBuilder; use rustfs_protos::proto_gen::node_service::{ BackgroundHealStatusRequest, CheckPartsRequest, DeleteBucketMetadataRequest, DeleteBucketRequest, DeletePathsRequest, DeletePolicyRequest, DeleteRequest, DeleteServiceAccountRequest, DeleteUserRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, DownloadProfileDataRequest, GenerallyLockRequest, GetAllBucketStatsRequest, GetBucketInfoRequest, GetBucketStatsDataRequest, GetCpusRequest, GetMemInfoRequest, - GetMetacacheListingRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, - GetSeLinuxInfoRequest, GetSrMetricsDataRequest, GetSysConfigRequest, GetSysErrorsRequest, HealBucketRequest, - ListBucketRequest, ListDirRequest, ListVolumesRequest, LoadBucketMetadataRequest, LoadGroupRequest, + GetMetacacheListingRequest, GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, + GetProcInfoRequest, GetSeLinuxInfoRequest, GetSrMetricsDataRequest, GetSysConfigRequest, GetSysErrorsRequest, + HealBucketRequest, ListBucketRequest, ListDirRequest, ListVolumesRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest, - MakeVolumesRequest, PingRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, - ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, - ServerInfoRequest, SignalServiceRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, - UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, - WriteRequest, node_service_client::NodeServiceClient, node_service_server::NodeServiceServer, + MakeVolumesRequest, Mss, PingRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest, ReadVersionRequest, + ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest, + RenamePartRequest, ServerInfoRequest, SignalServiceRequest, StartProfilingRequest, StatVolumeRequest, + StopRebalanceRequest, UpdateMetacacheListingRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, + WriteMetadataRequest, WriteRequest, node_service_client::NodeServiceClient, node_service_server::NodeServiceServer, }; + use std::collections::HashMap; use tokio::net::TcpListener; use tokio_stream::wrappers::TcpListenerStream; + use tonic::{Request, Response, Status}; fn create_test_node_service() -> NodeService { make_server() @@ -2527,7 +2527,7 @@ mod tests { #[test] fn test_background_rebalance_start_error_message_formats_error() { - let message = background_rebalance_start_error_message(Err(super::super::super::Error::other("boom"))) + let message = background_rebalance_start_error_message(Err(Error::other("boom"))) .expect("background rebalance start failure should be formatted"); assert!(message.contains("start_rebalance failed")); @@ -2536,7 +2536,7 @@ mod tests { #[test] fn test_stop_rebalance_response_reports_local_stop_error() { - let response = stop_rebalance_response(Err(super::super::super::Error::other("boom"))); + let response = stop_rebalance_response(Err(Error::other("boom"))); assert!(!response.success); assert!(response.error_info.as_deref().is_some_and(|message| message.contains("boom"))); @@ -2872,7 +2872,7 @@ mod tests { let mut vars = HashMap::new(); vars.insert(PEER_RESTSIGNAL.to_string(), SERVICE_SIGNAL_RELOAD_DYNAMIC.to_string()); - vars.insert(PEER_RESTSUB_SYS.to_string(), super::super::super::STORAGE_CLASS_SUB_SYS.to_string()); + vars.insert(PEER_RESTSUB_SYS.to_string(), STORAGE_CLASS_SUB_SYS.to_string()); let request = Request::new(SignalServiceRequest { vars: Some(Mss { value: vars }), diff --git a/rustfs/src/storage/runtime_sources.rs b/rustfs/src/storage/runtime_sources.rs index 0c067faaf..f92c54699 100644 --- a/rustfs/src/storage/runtime_sources.rs +++ b/rustfs/src/storage/runtime_sources.rs @@ -14,7 +14,7 @@ use crate::config::RustFSBufferConfig; use crate::runtime_sources as root_runtime_sources; -use crate::storage::ECStore; +use crate::storage::storage_api::runtime_sources_consumer::ECStore; use rustfs_credentials::Credentials; use rustfs_iam::{error::Result as IamResult, store::object::ObjectStore, sys::IamSys}; use rustfs_io_metrics::{PerformanceMetrics, internode_metrics::InternodeMetrics}; diff --git a/rustfs/src/storage/s3_api/bucket.rs b/rustfs/src/storage/s3_api/bucket.rs index 5ebaa5ef2..85161d295 100644 --- a/rustfs/src/storage/s3_api/bucket.rs +++ b/rustfs/src/storage/s3_api/bucket.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::contract::{ +use crate::storage::s3_api::common::rustfs_owner; +use crate::storage::storage_api::s3_api_consumer::bucket::contract::{ bucket::BucketInfo, list::{ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info}, }; -use crate::storage::s3_api::common::rustfs_owner; -use crate::storage::to_s3s_etag; +use crate::storage::storage_api::s3_api_consumer::bucket::to_s3s_etag; use percent_encoding::percent_decode_str; use s3s::dto::{ Bucket, CommonPrefix, DeleteMarkerEntry, EncodingType, ListBucketsOutput, ListObjectVersionsOutput, ListObjectsOutput, @@ -27,7 +27,7 @@ use s3s::{S3Error, S3ErrorCode}; use tracing::debug; use urlencoding::encode; -use crate::storage::StorageObjectInfo as ObjectInfo; +use crate::storage::storage_api::s3_api_consumer::bucket::StorageObjectInfo as ObjectInfo; const S3_MAX_KEYS: i32 = 1000; @@ -395,9 +395,9 @@ mod tests { build_list_object_versions_output, build_list_objects_output, build_list_objects_v2_output, parse_list_object_versions_params, parse_list_objects_v2_params, }; - use crate::storage::StorageObjectInfo as ObjectInfo; - use crate::storage::contract::bucket::BucketInfo; use crate::storage::s3_api::common::rustfs_owner; + use crate::storage::storage_api::s3_api_consumer::bucket::StorageObjectInfo as ObjectInfo; + use crate::storage::storage_api::s3_api_consumer::bucket::contract::bucket::BucketInfo; use s3s::S3ErrorCode; use s3s::dto::{CommonPrefix, EncodingType, ListObjectsV2Output, Object}; use time::OffsetDateTime; diff --git a/rustfs/src/storage/s3_api/multipart.rs b/rustfs/src/storage/s3_api/multipart.rs index 308aee7bd..83aaeaf47 100644 --- a/rustfs/src/storage/s3_api/multipart.rs +++ b/rustfs/src/storage/s3_api/multipart.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::storage::contract::multipart::{ListMultipartsInfo, ListPartsInfo}; use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; -use crate::storage::to_s3s_etag; +use crate::storage::storage_api::s3_api_consumer::multipart::contract::multipart::{ListMultipartsInfo, ListPartsInfo}; +use crate::storage::storage_api::s3_api_consumer::multipart::to_s3s_etag; use s3s::dto::{CommonPrefix, ListMultipartUploadsOutput, ListPartsOutput, MultipartUpload, Part, Timestamp}; use s3s::{S3Error, S3ErrorCode}; @@ -195,9 +195,11 @@ mod tests { MAX_MULTIPART_UPLOADS_LIST, build_list_multipart_uploads_output, build_list_parts_output, parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number, }; - use crate::storage::contract::multipart::{ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo}; use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; - use crate::storage::to_s3s_etag; + use crate::storage::storage_api::s3_api_consumer::multipart::contract::multipart::{ + ListMultipartsInfo, ListPartsInfo, MultipartInfo, PartInfo, + }; + use crate::storage::storage_api::s3_api_consumer::multipart::to_s3s_etag; use s3s::S3ErrorCode; use s3s::dto::Timestamp; use time::OffsetDateTime; diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 305168b20..0762fb88f 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -70,7 +70,7 @@ //! ``` use super::StorageError; -use crate::storage::runtime_sources; +use crate::storage::storage_api::runtime_sources_consumer::runtime_sources; #[cfg(feature = "rio-v2")] use aes_gcm::aead::Payload; use aes_gcm::{ @@ -2402,10 +2402,38 @@ fn ssec_invalid_request(message: &str) -> ApiError { } #[cfg(test)] +#[allow(unused_imports)] mod tests { - use super::*; - use http::HeaderValue; + #[cfg(feature = "rio-v2")] + use super::{ + DARE_CIPHER_AES_256_GCM, DARE_CIPHER_CHACHA20_POLY1305, MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM, SEALED_KEY_IV_SIZE, + SEALED_KEY_SIZE, is_legacy_rustfs_managed_metadata, is_supported_sealed_object_key_cipher, + }; + use super::{ + DecryptionRequest, EncryptionKeyKind, EncryptionMaterial, EncryptionRequest, INTERNAL_ENCRYPTION_ALGORITHM_HEADER, + INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER, KmsSseDekProvider, + MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER, MINIO_INTERNAL_ENCRYPTION_IV_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER, + MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_MULTIPART_HEADER, + MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER, + PrepareEncryptionRequest, SSEC_ORIGINAL_SIZE_HEADER, SSEType, SseDekProvider, SsecParams, StorageError, + TestSseDekProvider, encryption_material_to_metadata, extract_server_side_encryption_from_headers, + extract_ssec_params_from_headers, extract_ssekms_context_from_headers, generate_ssec_nonce, is_managed_sse, + map_get_object_reader_error, mark_encrypted_multipart_metadata, normalize_managed_metadata, reset_sse_dek_provider, + resolve_effective_kms_key_id, sse_decryption, sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, + validate_sse_headers_for_read, validate_sse_headers_for_write, validate_ssec_for_read, validate_ssec_params, + verify_ssec_key_match, + }; + use aes_gcm::aead::{Aead, KeyInit}; + use aes_gcm::{Aes256Gcm, Key, Nonce}; + use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD}; + use http::{HeaderMap, HeaderValue}; + use rustfs_kms::types::ObjectEncryptionContext; use rustfs_rio::{DecryptReader, EncryptReader}; + use rustfs_utils::http::headers::AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT; + use s3s::S3ErrorCode; + use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, ServerSideEncryption}; + use std::collections::HashMap; use std::sync::OnceLock; use temp_env::async_with_vars; use tokio::sync::Mutex; diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 81ff95a93..793c1898c 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -60,6 +60,246 @@ pub(crate) type StorageObjectInfo = super::ObjectInfo; pub(crate) type StorageObjectOptions = super::ObjectOptions; pub(crate) type StorageObjectToDelete = contract::object::ObjectToDelete; pub(crate) type StoragePutObjReader = super::PutObjReader; +pub(crate) use super::ecfs_extend::{ + RFC1123, apply_bucket_default_lock_retention, apply_cors_headers, check_preconditions, get_buffer_size_opt_in, + get_validated_store, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention, + parse_part_number_i32_to_usize, process_lambda_configurations, process_queue_configurations, process_topic_configurations, + remove_object_lock_metadata_for_copy, validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter, + validate_object_key, wrap_response_with_cors, +}; +pub(crate) use super::sse::{ + DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, extract_server_side_encryption_from_headers, sse_decryption, + sse_encryption, sse_prepare_encryption, strip_managed_encryption_metadata, validate_sse_headers_for_read, + validate_sse_headers_for_write, validate_ssec_for_read, +}; + +pub(crate) mod access_consumer { + pub(crate) use super::super::access::{ + PostObjectRequestMarker, ReqInfo, authorize_request, has_bypass_governance_header, req_info_mut, req_info_ref, + }; + + pub(crate) mod contract { + pub(crate) mod bucket { + pub(crate) use super::super::super::contract::bucket::BucketOperations; + } + } +} + +pub(crate) mod concurrency_consumer { + pub(crate) use super::super::concurrency::{ + ConcurrencyManager, GetObjectGuard, IoQueueStatus, IoStrategy, PutObjectGuard, get_concurrency_aware_buffer_size, + get_concurrency_manager, get_put_concurrency_aware_buffer_size, + }; +} + +pub(crate) mod deadlock_detector_consumer { + #[cfg(test)] + pub(crate) use super::super::deadlock_detector::RequestHangDetectionPolicy; + pub(crate) use super::super::deadlock_detector::{DeadlockDetector, get_deadlock_detector}; +} + +pub(crate) mod ecfs_consumer { + pub(crate) mod contract { + pub(crate) mod bucket { + pub(crate) use super::super::super::contract::bucket::{BucketOperations, BucketOptions}; + } + + pub(crate) mod object { + pub(crate) use super::super::super::contract::object::{ObjectLockRetentionOptions, ObjectOperations}; + } + } + + pub(crate) mod object_lock { + pub(crate) use super::super::super::{ + parse_object_lock_legal_hold, parse_object_lock_retention, validate_bucket_object_lock_enabled, + }; + } + + pub(crate) type StorageObjectOptions = super::StorageObjectOptions; +} + +pub(crate) mod ecfs_extend_consumer { + pub(crate) mod contract { + pub(crate) mod bucket { + pub(crate) use super::super::super::contract::bucket::{BucketOperations, BucketOptions}; + } + + pub(crate) mod object { + pub(crate) use super::super::super::contract::object::ObjectToDelete; + } + } + + pub(crate) type StorageObjectInfo = super::StorageObjectInfo; +} + +pub(crate) mod head_prefix_consumer { + pub(crate) use super::super::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children}; + + pub(crate) mod contract { + pub(crate) mod list { + pub(crate) use super::super::super::contract::list::ListOperations; + } + } +} + +pub(crate) mod helper_consumer { + pub(crate) use super::super::helper::{OperationHelper, spawn_background_with_context}; + + pub(crate) type StorageObjectInfo = super::StorageObjectInfo; +} + +pub(crate) mod options_consumer { + pub(crate) use super::super::options::{ + copy_dst_opts, copy_src_opts, del_opts, extract_metadata, extract_metadata_from_mime, + extract_metadata_from_mime_with_object_name, filter_object_metadata, get_complete_multipart_upload_opts, + get_content_sha256_with_query, get_opts, normalize_content_encoding_for_storage, parse_copy_source_range, put_opts, + validate_archive_content_encoding, + }; + + pub(crate) mod contract { + pub(crate) mod object { + pub(crate) use super::super::super::contract::object::HTTPPreconditions; + } + + pub(crate) mod range { + pub(crate) use super::super::super::contract::range::HTTPRangeSpec; + } + } + + pub(crate) type StorageObjectOptions = super::StorageObjectOptions; +} + +pub(crate) mod request_context_consumer { + pub(crate) use super::super::request_context::{ + RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced, + }; +} + +pub(crate) mod rpc_consumer { + pub(crate) use super::super::rpc::InternodeRpcService; + + pub(crate) mod http_service { + pub(crate) const DEFAULT_READ_BUFFER_SIZE: usize = super::super::DEFAULT_READ_BUFFER_SIZE; + pub(crate) use super::super::{StorageDiskRpcExt, WalkDirOptions, find_local_disk_by_ref, verify_rpc_signature}; + } + + pub(crate) mod node_service { + pub(crate) use super::super::{ + CollectMetricsOpts, DeleteOptions, DiskError, DiskInfoOptions, DiskStore, ECStore, Error, FileInfoVersions, + LocalPeerS3Client, MetricType, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, ReadMultipleReq, ReadMultipleResp, ReadOptions, + SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, StorageDiskRpcExt, StoragePeerS3ClientExt, + UpdateMetadataOpts, all_local_disk_path, collect_local_metrics, find_local_disk_by_ref, get_local_server_property, + load_bucket_metadata, reload_transition_tier_config, resolve_object_store_handle, set_bucket_metadata, + }; + pub(crate) type StorageResult = super::super::Result; + + #[cfg(test)] + pub(crate) const STORAGE_CLASS_SUB_SYS: &str = super::super::STORAGE_CLASS_SUB_SYS; + + pub(crate) mod contract { + pub(crate) mod admin { + pub(crate) use super::super::super::super::contract::admin::StorageAdminApi; + } + + pub(crate) mod bucket { + pub(crate) use super::super::super::super::contract::bucket::{ + BucketOptions, DeleteBucketOptions, MakeBucketOptions, + }; + } + } + } +} + +pub(crate) mod runtime_sources_consumer { + pub(crate) type ECStore = super::ECStore; + pub(crate) use crate::storage::runtime_sources; +} + +pub(crate) mod s3_api_consumer { + pub(crate) mod bucket { + pub(crate) use super::super::super::s3_api::bucket::{ + ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output, + build_list_objects_output, build_list_objects_v2_output, parse_list_object_versions_params, + parse_list_objects_v2_params, + }; + + pub(crate) mod contract { + pub(crate) mod bucket { + pub(crate) use super::super::super::super::contract::bucket::BucketInfo; + } + + pub(crate) mod list { + pub(crate) use super::super::super::super::contract::list::{ListObjectVersionsInfo, ListObjectsV2Info}; + } + } + + pub(crate) type StorageObjectInfo = super::super::StorageObjectInfo; + + pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag { + super::super::to_s3s_etag(etag) + } + } + + pub(crate) mod common { + pub(crate) use super::super::super::s3_api::common::rustfs_owner; + } + + pub(crate) mod multipart { + pub(crate) use super::super::super::s3_api::multipart::{ + ListMultipartUploadsParams, build_list_multipart_uploads_output, build_list_parts_output, + parse_list_multipart_uploads_params, parse_list_parts_params, parse_upload_part_number, + }; + + pub(crate) mod contract { + pub(crate) mod multipart { + pub(crate) use super::super::super::super::contract::multipart::{ListMultipartsInfo, ListPartsInfo}; + + #[cfg(test)] + pub(crate) use super::super::super::super::contract::multipart::{MultipartInfo, PartInfo}; + } + } + + pub(crate) fn to_s3s_etag(etag: &str) -> s3s::dto::ETag { + super::super::to_s3s_etag(etag) + } + } +} + +pub(crate) mod sse_consumer { + pub(crate) use super::super::sse::{ + EncryptionKeyKind, SSEType, build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers, + extract_ssekms_context_from_headers, map_get_object_reader_error, mark_encrypted_multipart_metadata, + }; + pub(crate) use super::{ + DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, apply_bucket_default_lock_retention, + extract_server_side_encryption_from_headers, get_buffer_size_opt_in, sse_decryption, sse_encryption, + sse_prepare_encryption, + }; +} + +pub(crate) mod timeout_wrapper_consumer { + pub(crate) use super::super::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper}; +} + +pub(crate) mod tonic_service_consumer { + pub(crate) use super::super::tonic_service::make_server; +} + +#[cfg(test)] +pub(crate) mod test_consumer { + pub(crate) use super::super::{ + apply_cors_headers, apply_default_lock_retention_metadata, check_preconditions, decode_tags_to_map, + get_adaptive_buffer_size_with_profile, get_buffer_size_opt_in, is_etag_equal, matches_origin_pattern, parse_etag, + parse_object_lock_legal_hold, parse_object_lock_retention, process_lambda_configurations, process_queue_configurations, + process_topic_configurations, remove_object_lock_metadata_for_copy, remove_object_lock_retention_metadata, + validate_bucket_object_lock_enabled, validate_list_object_unordered_with_delimiter, + }; + pub(crate) use super::{ + BucketMetadata, DEFAULT_READ_BUFFER_SIZE, bucket_metadata_sys_initialized, get_global_bucket_metadata_sys, + set_bucket_metadata, + }; + pub(crate) type StorageObjectInfo = super::StorageObjectInfo; +} pub(crate) mod ecstore_admin { pub(crate) use rustfs_ecstore::api::admin::{get_local_server_property, get_server_info}; diff --git a/rustfs/src/storage/timeout_wrapper.rs b/rustfs/src/storage/timeout_wrapper.rs index c5024812b..582c4c7e4 100644 --- a/rustfs/src/storage/timeout_wrapper.rs +++ b/rustfs/src/storage/timeout_wrapper.rs @@ -462,8 +462,11 @@ pub fn get_io_buffer_size() -> usize { } #[cfg(test)] +#[allow(unused_imports)] mod tests { - use super::*; + use super::{ + GetObjectTimeoutPolicy, RequestTimeoutWrapper, TimedGetObjectResult, get_duplex_buffer_size, get_io_buffer_size, + }; use std::time::Duration; #[test] diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index 35687cc8d..6f98f14d0 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -18,24 +18,39 @@ use rustfs_storage_api as storage_contracts; pub(crate) mod capacity { - pub(crate) use crate::storage::{all_local_disk, disk_drive_path, disk_endpoint}; + pub(crate) mod service { + pub(crate) use crate::storage::storage_api::{all_local_disk, disk_drive_path, disk_endpoint}; + } } pub(crate) mod cluster { - pub(crate) use super::storage_contracts::{ - CapabilitySnapshotError, CapabilityStatus, DiskCapabilities, MemorySamplingState, ObservabilitySnapshot, - ObservabilitySnapshotProvider, PlatformSupport, TopologyCapabilities, TopologySnapshot, TopologySnapshotProvider, - UserspaceProfilingCapability, - }; - pub(crate) use crate::storage::{EndpointServerPools, topology_snapshot_from_endpoint_pools_with_capabilities}; + pub(crate) mod contract { + pub(crate) mod capability { + #[cfg(test)] + pub(crate) use super::super::super::storage_contracts::CapabilityState; + pub(crate) use super::super::super::storage_contracts::{ + CapabilitySnapshotError, CapabilityStatus, MemorySamplingState, PlatformSupport, UserspaceProfilingCapability, + }; + } + + pub(crate) mod observability { + pub(crate) use super::super::super::storage_contracts::{ObservabilitySnapshot, ObservabilitySnapshotProvider}; + } + + pub(crate) mod topology { + pub(crate) use super::super::super::storage_contracts::{ + DiskCapabilities, TopologyCapabilities, TopologySnapshot, TopologySnapshotProvider, + }; + } + } + + pub(crate) use crate::storage::storage_api::{EndpointServerPools, topology_snapshot_from_endpoint_pools_with_capabilities}; #[cfg(test)] - pub(crate) use super::storage_contracts::CapabilityState; - #[cfg(test)] - pub(crate) use crate::storage::{Endpoint, Endpoints, PoolEndpoints}; + pub(crate) use crate::storage::storage_api::{Endpoint, Endpoints, PoolEndpoints}; pub(crate) mod control_plane { - pub(crate) use crate::storage::ecstore_cluster::{ + pub(crate) use crate::storage::storage_api::ecstore_cluster::{ ClusterControlPlane, ClusterControlPlaneSnapshot, ClusterLocalNodeStorageSnapshot, ClusterMembershipSnapshot, ClusterPeerHealthSnapshot, ClusterPoolStateSnapshot, }; @@ -44,103 +59,208 @@ pub(crate) mod cluster { #[cfg(test)] pub(crate) mod config_test { - pub(crate) use crate::storage::DisksLayout; + pub(crate) use crate::storage::storage_api::DisksLayout; } pub(crate) mod error { - pub(crate) use super::storage_contracts::HTTPRangeError; - pub(crate) use crate::storage::{QuotaError, StorageError}; + pub(crate) mod contract { + pub(crate) mod range { + pub(crate) use super::super::super::storage_contracts::HTTPRangeError; + } + } + + pub(crate) use crate::storage::storage_api::{QuotaError, StorageError}; } pub(crate) mod protocols { - pub(crate) mod access { - pub(crate) use crate::storage::access::ReqInfo; - } - - pub(crate) mod ecfs { - pub(crate) type FS = crate::storage::FS; - } - - pub(crate) mod request_context { - pub(crate) use crate::storage::request_context::RequestContext; + pub(crate) mod client { + pub(crate) use crate::storage::storage_api::access_consumer::ReqInfo; + pub(crate) type FS = crate::storage::storage_api::FS; + pub(crate) use crate::storage::storage_api::request_context_consumer::RequestContext; } } pub(crate) mod server { - pub(crate) use super::storage_contracts::StorageAdminApi; - pub(crate) use crate::storage::{ - ECStore, Endpoint, EndpointServerPools, Error, EventArgs, StorageObjectInfo, TONIC_RPC_PREFIX, apply_cors_headers, - is_dist_erasure, read_config, register_event_dispatch_hook, save_config, verify_rpc_signature, - }; + pub(crate) mod event { + pub(crate) mod contract { + #[cfg(test)] + pub(crate) mod lifecycle { + pub(crate) use super::super::super::super::storage_contracts::TransitionedObject; + } + } - #[cfg(test)] - pub(crate) use super::storage_contracts::TransitionedObject; - #[cfg(test)] - pub(crate) use crate::storage::{Endpoints, PoolEndpoints}; - - pub(crate) mod ecfs { - pub(crate) type FS = crate::storage::FS; + pub(crate) use crate::storage::storage_api::{EventArgs, StorageObjectInfo, register_event_dispatch_hook}; } - pub(crate) mod request_context { - pub(crate) use crate::storage::request_context::{ - RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced, - }; + pub(crate) mod http { + pub(crate) use crate::storage::storage_api::{TONIC_RPC_PREFIX, verify_rpc_signature}; + + pub(crate) mod ecfs { + pub(crate) type FS = crate::storage::storage_api::FS; + } + + pub(crate) mod request_context { + pub(crate) use crate::storage::storage_api::request_context_consumer::{ + RequestContext, extract_request_id_from_headers, + }; + } + + pub(crate) mod rpc { + pub(crate) use crate::storage::storage_api::rpc_consumer::InternodeRpcService; + } + + pub(crate) mod tonic_service { + pub(crate) use crate::storage::storage_api::tonic_service_consumer::make_server; + } } - pub(crate) mod rpc { - pub(crate) use crate::storage::rpc::InternodeRpcService; + pub(crate) mod layer { + pub(crate) use crate::storage::storage_api::apply_cors_headers; + + pub(crate) mod request_context { + pub(crate) use crate::storage::storage_api::request_context_consumer::{ + RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced, + }; + } } - pub(crate) mod tonic_service { - pub(crate) use crate::storage::tonic_service::make_server; + pub(crate) mod module_switch { + pub(crate) use crate::storage::storage_api::{Error, read_config, save_config}; + } + + pub(crate) mod readiness { + pub(crate) mod contract { + pub(crate) mod admin { + pub(crate) use super::super::super::super::storage_contracts::StorageAdminApi; + } + } + + pub(crate) use crate::storage::storage_api::{Endpoint, EndpointServerPools, is_dist_erasure}; + + #[cfg(test)] + pub(crate) use crate::storage::storage_api::{Endpoints, PoolEndpoints}; + } + + pub(crate) mod runtime_sources { + pub(crate) use crate::storage::storage_api::{ECStore, EndpointServerPools}; } } pub(crate) mod startup { - pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions}; - pub(crate) use crate::storage::{ - DynReplicationPool, ECStore, EndpointServerPools, Result, get_bucket_notification_config, init_background_replication, - init_bucket_metadata_sys, init_ecstore_config, init_global_config_sys, init_local_disks, init_lock_clients, - new_global_notification_sys, prewarm_local_disk_id_map, process_lambda_configurations, process_queue_configurations, - process_topic_configurations, set_global_endpoints, set_global_region, set_global_rustfs_port, - set_workload_admission_snapshot_provider, shutdown_background_services, try_migrate_bucket_metadata, - try_migrate_iam_config, try_migrate_server_config, update_erasure_type, - }; + pub(crate) mod background { + pub(crate) use crate::storage::storage_api::{ECStore, set_workload_admission_snapshot_provider}; + } - #[cfg(test)] - pub(crate) use crate::storage::Error; + pub(crate) mod bucket_metadata { + pub(crate) mod contract { + pub(crate) mod bucket { + pub(crate) use super::super::super::super::storage_contracts::{BucketOperations, BucketOptions}; + } + } + + pub(crate) use crate::storage::storage_api::{ + ECStore, init_bucket_metadata_sys, try_migrate_bucket_metadata, try_migrate_iam_config, + }; + } + + pub(crate) mod deadlock { + pub(crate) use crate::storage::storage_api::deadlock_detector_consumer::get_deadlock_detector; + } + + pub(crate) mod fs_guard { + pub(crate) use crate::storage::storage_api::EndpointServerPools; + } + + pub(crate) mod iam { + pub(crate) use crate::storage::storage_api::ECStore; + } + + pub(crate) mod init { + pub(crate) use crate::storage::storage_api::{ + get_bucket_notification_config, process_lambda_configurations, process_queue_configurations, + process_topic_configurations, + }; + + pub(crate) mod concurrency { + pub(crate) use crate::storage::storage_api::concurrency_consumer::get_concurrency_manager; + } + + pub(crate) mod ecfs { + pub(crate) type FS = crate::storage::storage_api::FS; + } + } + + pub(crate) mod lifecycle { + pub(crate) use crate::storage::storage_api::ECStore; + } + + pub(crate) mod notification { + pub(crate) use crate::storage::storage_api::{EndpointServerPools, Result, new_global_notification_sys}; + + #[cfg(test)] + pub(crate) use crate::storage::storage_api::Error; + } + + pub(crate) mod runtime_sources { + pub(crate) use crate::storage::storage_api::{DynReplicationPool, set_global_region, set_global_rustfs_port}; + } + + pub(crate) mod services { + pub(crate) use crate::storage::storage_api::{ECStore, EndpointServerPools}; + } + + pub(crate) mod shutdown { + pub(crate) use crate::storage::storage_api::shutdown_background_services; + } + + pub(crate) mod storage { + pub(crate) use crate::storage::storage_api::{ + ECStore, EndpointServerPools, init_background_replication, init_ecstore_config, init_global_config_sys, + init_local_disks, init_lock_clients, prewarm_local_disk_id_map, set_global_endpoints, try_migrate_server_config, + update_erasure_type, + }; + } +} + +pub(crate) mod workload { + pub(crate) mod admission { + pub(crate) use crate::storage::storage_api::{bucket_metadata_runtime_initialized, replication_queue_current_count}; + } pub(crate) mod concurrency { - pub(crate) use crate::storage::concurrency::get_concurrency_manager; - } - - pub(crate) mod deadlock_detector { - pub(crate) use crate::storage::deadlock_detector::get_deadlock_detector; - } - - pub(crate) mod ecfs { - pub(crate) type FS = crate::storage::FS; + pub(crate) use crate::storage::storage_api::concurrency_consumer::get_concurrency_manager; } } pub(crate) mod table { - pub(crate) use super::storage_contracts::{ - HTTPPreconditions, HTTPRangeSpec, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, NamespaceLocking, ObjectIO, - ObjectInfoOrErr, ObjectOperations, WalkOptions, - }; - pub(crate) use crate::storage::{ + pub(crate) mod contract { + pub(crate) mod http { + pub(crate) use super::super::super::storage_contracts::HTTPPreconditions; + } + + pub(crate) mod list { + pub(crate) use super::super::super::storage_contracts::{ + ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr, WalkOptions, + }; + } + + pub(crate) mod namespace { + pub(crate) use super::super::super::storage_contracts::NamespaceLocking; + } + + pub(crate) mod object { + pub(crate) use super::super::super::storage_contracts::{ObjectIO, ObjectOperations}; + } + + pub(crate) mod range { + pub(crate) use super::super::super::storage_contracts::HTTPRangeSpec; + } + } + + pub(crate) use crate::storage::storage_api::{ BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG, BUCKET_TABLE_RESERVED_PREFIX, Error, RUSTFS_META_BUCKET, StorageDeletedObject, StorageError, StorageGetObjectReader, StorageObjectInfo, StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, get_bucket_metadata, get_lock_acquire_timeout, table_catalog_path_hash, }; } - -pub(crate) mod workload { - pub(crate) use crate::storage::{bucket_metadata_runtime_initialized, replication_queue_current_count}; - - pub(crate) mod concurrency { - pub(crate) use crate::storage::concurrency::get_concurrency_manager; - } -} diff --git a/rustfs/src/table_catalog.rs b/rustfs/src/table_catalog.rs index 537a15841..d36aa5735 100644 --- a/rustfs/src/table_catalog.rs +++ b/rustfs/src/table_catalog.rs @@ -27,13 +27,18 @@ use std::{ time::{Duration as StdDuration, Instant}, }; +use crate::storage_api::table::contract::http::HTTPPreconditions; +use crate::storage_api::table::contract::list::{ + ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, + ListOperations as StorageListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions, +}; +use crate::storage_api::table::contract::namespace::NamespaceLocking as StorageNamespaceLocking; +use crate::storage_api::table::contract::object::{ObjectIO as StorageObjectIO, ObjectOperations as StorageObjectOperations}; +use crate::storage_api::table::contract::range::HTTPRangeSpec; use crate::storage_api::table::{ BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, BUCKET_TABLE_CONFIG, - BUCKET_TABLE_RESERVED_PREFIX, Error as EcstoreError, HTTPPreconditions, HTTPRangeSpec, - ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, - ListOperations as StorageListOperations, NamespaceLocking as StorageNamespaceLocking, ObjectIO as StorageObjectIO, - ObjectInfoOrErr as StorageObjectInfoOrErr, ObjectOperations as StorageObjectOperations, RUSTFS_META_BUCKET, StorageError, - WalkOptions as StorageWalkOptions, get_bucket_metadata, get_lock_acquire_timeout, table_catalog_path_hash, + BUCKET_TABLE_RESERVED_PREFIX, Error as EcstoreError, RUSTFS_META_BUCKET, StorageError, get_bucket_metadata, + get_lock_acquire_timeout, table_catalog_path_hash, }; use bytes::Bytes; use datafusion::{ diff --git a/rustfs/src/workload_admission.rs b/rustfs/src/workload_admission.rs index 1bd8d3dca..65deaa02a 100644 --- a/rustfs/src/workload_admission.rs +++ b/rustfs/src/workload_admission.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::runtime_sources::resolve_replication_pool_handle; -use crate::storage_api::workload::{bucket_metadata_runtime_initialized, replication_queue_current_count}; +use crate::storage_api::workload::admission::{bucket_metadata_runtime_initialized, replication_queue_current_count}; use rustfs_concurrency::{ AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider, WorkloadClass, diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index 9bae8b4e9..c62860168 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -184,6 +184,7 @@ RUSTFS_ADMIN_RUNTIME_SOURCE_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_admin_runtime_so RUSTFS_APP_CONTEXT_DIRECT_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_context_direct_bypass_hits.txt" RUSTFS_APP_USECASE_RUNTIME_SOURCE_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_usecase_runtime_source_bypass_hits.txt" RUSTFS_APP_USECASE_STORAGE_WILDCARD_HITS_FILE="${TMP_DIR}/rustfs_app_usecase_storage_wildcard_hits.txt" +RUSTFS_APP_USECASE_CONTRACT_ROOT_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_app_usecase_contract_root_consumer_hits.txt" RUSTFS_APP_WILDCARD_IMPORT_HITS_FILE="${TMP_DIR}/rustfs_app_wildcard_import_hits.txt" RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_usecase_s3_api_bypass_hits.txt" RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_usecase_storage_api_bypass_hits.txt" @@ -192,6 +193,17 @@ RUSTFS_ADMIN_STORAGE_API_ROOT_FACADE_HITS_FILE="${TMP_DIR}/rustfs_admin_storage_ RUSTFS_ROOT_STORAGE_API_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_root_storage_api_bypass_hits.txt" RUSTFS_ROOT_STORAGE_API_CONTRACT_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_root_storage_api_contract_bypass_hits.txt" RUSTFS_ROOT_STORAGE_API_DOMAIN_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_root_storage_api_domain_bypass_hits.txt" +RUSTFS_ROOT_STORAGE_CONTRACT_ROOT_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_root_storage_contract_root_consumer_hits.txt" +RUSTFS_ROOT_RUNTIME_FACADE_ROOT_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_root_runtime_facade_root_consumer_hits.txt" +RUSTFS_STORAGE_OWNER_ROOT_FACADE_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_storage_owner_root_facade_consumer_hits.txt" +RUSTFS_STORAGE_OWNER_RUNTIME_ROOT_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_storage_owner_runtime_root_consumer_hits.txt" +RUSTFS_STORAGE_OWNER_HELPER_ROOT_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_storage_owner_helper_root_consumer_hits.txt" +RUSTFS_STORAGE_OWNER_RPC_ROOT_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_storage_owner_rpc_root_consumer_hits.txt" +RUSTFS_STORAGE_OWNER_WILDCARD_IMPORT_HITS_FILE="${TMP_DIR}/rustfs_storage_owner_wildcard_import_hits.txt" +RUSTFS_STORAGE_OWNER_ROOT_EXPORT_GLOB_HITS_FILE="${TMP_DIR}/rustfs_storage_owner_root_export_glob_hits.txt" +RUSTFS_STORAGE_FACADE_DIRECT_OWNER_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_storage_facade_direct_owner_bypass_hits.txt" +RUSTFS_STORAGE_OWNER_SSE_ROOT_EXPORT_HITS_FILE="${TMP_DIR}/rustfs_storage_owner_sse_root_export_hits.txt" +RUSTFS_STORAGE_OWNER_TEST_ROOT_CONSUMER_HITS_FILE="${TMP_DIR}/rustfs_storage_owner_test_root_consumer_hits.txt" RUSTFS_ADMIN_STORAGE_API_DOMAIN_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_admin_storage_api_domain_bypass_hits.txt" RUSTFS_APP_STORAGE_API_DOMAIN_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_storage_api_domain_bypass_hits.txt" RUSTFS_APP_STORAGE_API_CONTRACT_BYPASS_HITS_FILE="${TMP_DIR}/rustfs_app_storage_api_contract_bypass_hits.txt" @@ -1852,6 +1864,162 @@ if [[ -s "$RUSTFS_ROOT_STORAGE_API_DOMAIN_BYPASS_HITS_FILE" ]]; then report_failure "RustFS root storage-api consumers must use domain modules from rustfs/src/storage_api.rs: $(paste -sd '; ' "$RUSTFS_ROOT_STORAGE_API_DOMAIN_BYPASS_HITS_FILE")" fi +( + cd "$ROOT_DIR" + rg -n -U --with-filename \ + 'crate::storage_api::(?:capacity::(?:\{|all_local_disk|disk_drive_path|disk_endpoint)|protocols::(?:access|ecfs|request_context)\b|server::(?:\{|contract\b|ecfs\b|request_context\b|rpc\b|tonic_service\b|apply_cors_headers|ECStore|Endpoint(?:s|ServerPools)?|Error|EventArgs|StorageObjectInfo|TONIC_RPC_PREFIX|is_dist_erasure|read_config|register_event_dispatch_hook|save_config|verify_rpc_signature)|startup::(?:\{|contract\b|concurrency\b|deadlock_detector\b|ecfs\b|DynReplicationPool|ECStore|EndpointServerPools|Error\b|Result\b|get_bucket_notification_config|init_background_replication|init_bucket_metadata_sys|init_ecstore_config|init_global_config_sys|init_local_disks|init_lock_clients|new_global_notification_sys|prewarm_local_disk_id_map|process_lambda_configurations|process_queue_configurations|process_topic_configurations|set_global_endpoints|set_global_region|set_global_rustfs_port|set_workload_admission_snapshot_provider|shutdown_background_services|try_migrate_bucket_metadata|try_migrate_iam_config|try_migrate_server_config|update_erasure_type)|workload::(?:\{|bucket_metadata_runtime_initialized|replication_queue_current_count))' \ + rustfs/src \ + --glob '*.rs' \ + --glob '!rustfs/src/storage_api.rs' || true +) >"$RUSTFS_ROOT_RUNTIME_FACADE_ROOT_CONSUMER_HITS_FILE" + +if [[ -s "$RUSTFS_ROOT_RUNTIME_FACADE_ROOT_CONSUMER_HITS_FILE" ]]; then + report_failure "RustFS root runtime facade consumers must use consumer-domain modules from rustfs/src/storage_api.rs: $(paste -sd '; ' "$RUSTFS_ROOT_RUNTIME_FACADE_ROOT_CONSUMER_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n -U --with-filename \ + 'crate::storage::(?:contract::|StorageObjectInfo\b|StorageObjectOptions\b|to_s3s_etag\b)' \ + rustfs/src/storage \ + --glob '*.rs' \ + --glob '!rustfs/src/storage/storage_api.rs' || true +) >"$RUSTFS_STORAGE_OWNER_ROOT_FACADE_CONSUMER_HITS_FILE" + +if [[ -s "$RUSTFS_STORAGE_OWNER_ROOT_FACADE_CONSUMER_HITS_FILE" ]]; then + report_failure "RustFS storage-owner consumers must use storage_api consumer-domain modules instead of root storage facades: $(paste -sd '; ' "$RUSTFS_STORAGE_OWNER_ROOT_FACADE_CONSUMER_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n -U --with-filename \ + 'crate::storage::(?:runtime_sources\b|ECStore\b)' \ + rustfs/src/storage \ + --glob '*.rs' \ + --glob '!rustfs/src/storage/storage_api.rs' || true +) >"$RUSTFS_STORAGE_OWNER_RUNTIME_ROOT_CONSUMER_HITS_FILE" + +if [[ -s "$RUSTFS_STORAGE_OWNER_RUNTIME_ROOT_CONSUMER_HITS_FILE" ]]; then + report_failure "RustFS storage-owner runtime consumers must use storage_api consumer-domain modules instead of root storage runtime facades: $(paste -sd '; ' "$RUSTFS_STORAGE_OWNER_RUNTIME_ROOT_CONSUMER_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n -U --with-filename \ + 'crate::storage::(?:parse_object_lock_legal_hold\b|parse_object_lock_retention\b|validate_bucket_object_lock_enabled\b)' \ + rustfs/src/storage \ + --glob '*.rs' \ + --glob '!rustfs/src/storage/storage_api.rs' || true +) >"$RUSTFS_STORAGE_OWNER_HELPER_ROOT_CONSUMER_HITS_FILE" + +if [[ -s "$RUSTFS_STORAGE_OWNER_HELPER_ROOT_CONSUMER_HITS_FILE" ]]; then + report_failure "RustFS storage-owner helper consumers must use storage_api consumer-domain modules instead of root storage helpers: $(paste -sd '; ' "$RUSTFS_STORAGE_OWNER_HELPER_ROOT_CONSUMER_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n -U --with-filename \ + '(?:use\s+super::super::(?:StorageDiskRpcExt|WalkDirOptions|find_local_disk_by_ref|verify_rpc_signature)|super::super::Result<|super::super::super::(?:Error|STORAGE_CLASS_SUB_SYS))' \ + rustfs/src/storage/rpc/http_service.rs \ + rustfs/src/storage/rpc/node_service.rs || true +) >"$RUSTFS_STORAGE_OWNER_RPC_ROOT_CONSUMER_HITS_FILE" + +if [[ -s "$RUSTFS_STORAGE_OWNER_RPC_ROOT_CONSUMER_HITS_FILE" ]]; then + report_failure "RustFS storage-owner RPC consumers must use storage_api consumer-domain modules instead of relative root storage facades: $(paste -sd '; ' "$RUSTFS_STORAGE_OWNER_RPC_ROOT_CONSUMER_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n --with-filename \ + '^[[:space:]]*use super::\*;' \ + rustfs/src/storage \ + --glob '*.rs' || true +) >"$RUSTFS_STORAGE_OWNER_WILDCARD_IMPORT_HITS_FILE" + +if [[ -s "$RUSTFS_STORAGE_OWNER_WILDCARD_IMPORT_HITS_FILE" ]]; then + report_failure "RustFS storage-owner modules must use explicit imports instead of parent wildcard imports: $(paste -sd '; ' "$RUSTFS_STORAGE_OWNER_WILDCARD_IMPORT_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n --with-filename \ + '^pub\(crate\) use storage_api::\*;' \ + rustfs/src/storage/mod.rs || true +) >"$RUSTFS_STORAGE_OWNER_ROOT_EXPORT_GLOB_HITS_FILE" + +if [[ -s "$RUSTFS_STORAGE_OWNER_ROOT_EXPORT_GLOB_HITS_FILE" ]]; then + report_failure "RustFS storage owner root must explicitly list storage_api re-exports instead of using a wildcard: $(paste -sd '; ' "$RUSTFS_STORAGE_OWNER_ROOT_EXPORT_GLOB_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg --pcre2 -n --with-filename \ + '(use crate::storage::\{|crate::storage::(?!storage_api\b))' \ + rustfs/src/storage_api.rs \ + rustfs/src/app/storage_api.rs \ + rustfs/src/admin/storage_api.rs || true +) >"$RUSTFS_STORAGE_FACADE_DIRECT_OWNER_BYPASS_HITS_FILE" + +if [[ -s "$RUSTFS_STORAGE_FACADE_DIRECT_OWNER_BYPASS_HITS_FILE" ]]; then + report_failure "RustFS root/app/admin storage facades must consume owner APIs through crate::storage::storage_api: $(paste -sd '; ' "$RUSTFS_STORAGE_FACADE_DIRECT_OWNER_BYPASS_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n --with-filename \ + '^pub\(crate\) use sse::' \ + rustfs/src/storage/mod.rs || true +) >"$RUSTFS_STORAGE_OWNER_SSE_ROOT_EXPORT_HITS_FILE" + +if [[ -s "$RUSTFS_STORAGE_OWNER_SSE_ROOT_EXPORT_HITS_FILE" ]]; then + report_failure "RustFS storage owner root must expose SSE helpers through storage_api instead of root sse re-exports: $(paste -sd '; ' "$RUSTFS_STORAGE_OWNER_SSE_ROOT_EXPORT_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + rg -n -U --with-filename \ + 'use\s+(?:super::super|crate::storage)::\{[^}]*?(?:BucketMetadata|DEFAULT_READ_BUFFER_SIZE|bucket_metadata_sys_initialized|get_global_bucket_metadata_sys|set_bucket_metadata|apply_cors_headers|apply_default_lock_retention_metadata|check_preconditions|decode_tags_to_map|get_adaptive_buffer_size_with_profile|get_buffer_size_opt_in|is_etag_equal|matches_origin_pattern|parse_etag|parse_object_lock_legal_hold|parse_object_lock_retention|process_lambda_configurations|process_queue_configurations|process_topic_configurations|remove_object_lock_metadata_for_copy|remove_object_lock_retention_metadata|validate_bucket_object_lock_enabled|validate_list_object_unordered_with_delimiter)' \ + rustfs/src/storage/ecfs_test.rs || true +) >"$RUSTFS_STORAGE_OWNER_TEST_ROOT_CONSUMER_HITS_FILE" + +if [[ -s "$RUSTFS_STORAGE_OWNER_TEST_ROOT_CONSUMER_HITS_FILE" ]]; then + report_failure "RustFS storage-owner tests must use storage_api test consumer modules instead of root storage helper facades: $(paste -sd '; ' "$RUSTFS_STORAGE_OWNER_TEST_ROOT_CONSUMER_HITS_FILE")" +fi + +( + cd "$ROOT_DIR" + { + rg -n -U --with-filename \ + 'storage_api::cluster::\{[^}]*?(CapabilitySnapshotError|CapabilityState|CapabilityStatus|DiskCapabilities|MemorySamplingState|ObservabilitySnapshot|ObservabilitySnapshotProvider|PlatformSupport|TopologyCapabilities|TopologySnapshot|TopologySnapshotProvider|UserspaceProfilingCapability)' \ + rustfs/src \ + --glob '*.rs' \ + --glob '!rustfs/src/storage_api.rs' \ + --glob '!rustfs/src/admin/**' \ + --glob '!rustfs/src/app/**' \ + --glob '!rustfs/src/storage/**' || true + rg -n --with-filename \ + 'storage_api::cluster::(?:CapabilitySnapshotError|CapabilityState|CapabilityStatus|DiskCapabilities|MemorySamplingState|ObservabilitySnapshot|ObservabilitySnapshotProvider|PlatformSupport|TopologyCapabilities|TopologySnapshot|TopologySnapshotProvider|UserspaceProfilingCapability)\b|storage_api::server::(?:StorageAdminApi|TransitionedObject)\b|storage_api::startup::(?:BucketOperations|BucketOptions)\b|storage_api::table::(?:HTTPPreconditions|HTTPRangeSpec|ListObjectVersionsInfo|ListObjectsV2Info|ListOperations|NamespaceLocking|ObjectIO|ObjectInfoOrErr|ObjectOperations|WalkOptions)\b|storage_api::error::HTTPRangeError\b' \ + rustfs/src \ + --glob '*.rs' \ + --glob '!rustfs/src/storage_api.rs' \ + --glob '!rustfs/src/admin/**' \ + --glob '!rustfs/src/app/**' \ + --glob '!rustfs/src/storage/**' || true + rg -n -U --with-filename \ + 'storage_api::(?:server::\{[^}]*?(StorageAdminApi|TransitionedObject)|startup::\{[^}]*?(BucketOperations|BucketOptions)|table::\{[^}]*?(HTTPPreconditions|HTTPRangeSpec|ListObjectVersionsInfo|ListObjectsV2Info|ListOperations|NamespaceLocking|ObjectIO|ObjectInfoOrErr|ObjectOperations|WalkOptions)|error::\{[^}]*?HTTPRangeError)' \ + rustfs/src \ + --glob '*.rs' \ + --glob '!rustfs/src/storage_api.rs' \ + --glob '!rustfs/src/admin/**' \ + --glob '!rustfs/src/app/**' \ + --glob '!rustfs/src/storage/**' || true + } +) >"$RUSTFS_ROOT_STORAGE_CONTRACT_ROOT_CONSUMER_HITS_FILE" + +if [[ -s "$RUSTFS_ROOT_STORAGE_CONTRACT_ROOT_CONSUMER_HITS_FILE" ]]; then + report_failure "RustFS root storage consumers must import storage contracts from domain modules, not root storage_api facades: $(paste -sd '; ' "$RUSTFS_ROOT_STORAGE_CONTRACT_ROOT_CONSUMER_HITS_FILE")" +fi + ( cd "$ROOT_DIR" rg -n --with-filename \ @@ -1877,6 +2045,31 @@ if [[ -s "$RUSTFS_APP_STORAGE_API_CONTRACT_BYPASS_HITS_FILE" ]]; then report_failure "RustFS app storage contracts must stay behind rustfs/src/app/storage_api.rs: $(paste -sd '; ' "$RUSTFS_APP_STORAGE_API_CONTRACT_BYPASS_HITS_FILE")" fi +( + cd "$ROOT_DIR" + { + rg -n --with-filename \ + 'storage_api::(?:admin_usecase|bucket_usecase|object_usecase|multipart_usecase|select_object)::(?:StorageAdminApi|BucketOperations|BucketOptions|DeleteBucketOptions|MakeBucketOptions|ListObjectVersionsInfo|ListObjectsV2Info|ListOperations|HTTPPreconditions|HTTPRangeSpec|NamespaceLocking|ObjectIO|ObjectOperations|CompletePart|MultipartOperations|MultipartUploadResult)\b' \ + rustfs/src/app \ + --glob '*.rs' \ + --glob '!storage_api.rs' || true + rg -n -U --with-filename \ + 'storage_api::test::\{[^}]*?(BucketOperations|BucketOptions|HealOperations|ListOperations|MakeBucketOptions|MultipartOperations|ObjectIO|ObjectOperations)' \ + rustfs/src/app \ + --glob '*.rs' \ + --glob '!storage_api.rs' || true + rg -n --with-filename \ + 'storage_api::test::(?:BucketOperations|BucketOptions|HealOperations|ListOperations|MakeBucketOptions|MultipartOperations|ObjectIO|ObjectOperations)\b' \ + rustfs/src/app \ + --glob '*.rs' \ + --glob '!storage_api.rs' || true + } +) >"$RUSTFS_APP_USECASE_CONTRACT_ROOT_CONSUMER_HITS_FILE" + +if [[ -s "$RUSTFS_APP_USECASE_CONTRACT_ROOT_CONSUMER_HITS_FILE" ]]; then + report_failure "RustFS app usecase modules must import storage contracts from domain modules, not usecase root facades: $(paste -sd '; ' "$RUSTFS_APP_USECASE_CONTRACT_ROOT_CONSUMER_HITS_FILE")" +fi + ( cd "$ROOT_DIR" rg -n --with-filename \ diff --git a/scripts/layer-dependency-baseline.txt b/scripts/layer-dependency-baseline.txt index 94773a74a..4539e6f75 100644 --- a/scripts/layer-dependency-baseline.txt +++ b/scripts/layer-dependency-baseline.txt @@ -21,22 +21,6 @@ cycle|infra<->interface dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_bucket_meta_hook dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_delete_bucket_hook dep|rustfs/src/app/bucket_usecase.rs|app->interface|crate::admin::handlers::site_replication::site_replication_make_bucket_hook -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::ecfs::FS -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::ListObjectVersionsParams -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::ListObjectsV2Params -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::build_list_buckets_output -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::build_list_object_versions_output -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::build_list_objects_output -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::build_list_objects_v2_output -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::parse_list_object_versions_params -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::bucket::parse_list_objects_v2_params -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::common::rustfs_owner -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::multipart::ListMultipartUploadsParams -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::multipart::build_list_multipart_uploads_output -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::multipart::build_list_parts_output -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::multipart::parse_list_multipart_uploads_params -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::multipart::parse_list_parts_params -dep|rustfs/src/app/storage_api.rs|app->interface|crate::storage::s3_api::multipart::parse_upload_part_number dep|rustfs/src/init.rs|infra->interface|crate::admin dep|rustfs/src/runtime_sources.rs|infra->app|crate::app::context dep|rustfs/src/server/http.rs|infra->interface|crate::admin