Compare commits

..

7 Commits

Author SHA1 Message Date
overtrue 4399c7e194 fix(app): keep relative paths valid in moved multipart code
Code that moves from multipart_usecase.rs into an operation file is one
module deeper, so its super:: paths gain a super:: (the codemod now does
this) and the regenerated split carries the result.
2026-11-11 14:45:24 +08:00
overtrue c5885b328d refactor(app): split multipart_usecase.rs into one file per operation
Mechanical move produced by scripts/codemods/split_multipart_usecase.py:
no signature, behaviour or test-body change. The struct, its
constructors and the shared helpers stay in multipart_usecase.rs; each
operation, its helpers and its tests move to multipart_usecase/<op>.rs.
New files reach s3s only through crate::app::storage_api::s3, which
gains the dto and xml re-exports; the nextest filter that pins one test
follows it to its new module path.
2026-11-11 14:21:24 +08:00
overtrue 682be074cd chore(app): add codemod that splits the multipart use case per operation 2026-11-11 14:21:24 +08:00
overtrue 3e8f7a001f merge: bring in guard source-set readers from #9292 2026-11-11 14:11:59 +08:00
Chris An 8172df44ec fix(ci): preserve guard and reference parity tests on merge 2026-11-10 22:55:10 -05:00
overtrue 7b3701ddc6 test(guards): avoid a typos false positive in a fixture 2026-11-11 10:40:32 +08:00
overtrue 2df55c6dea test(guards): read file plus split directory in text guards
Text guards that read one file with include_str! or by path broke, or silently stopped seeing the code they guard, when that code moved to another file. They now read the file they read today, which stays required, plus the sibling directory a mechanical split will populate (storage/legacy_edge, app/bucket_usecase, app/multipart_usecase, app/lifecycle_transition_api_test). Counts are summed over the set and split markers must occur exactly once across it. Refs rustfs/backlog#2765.
2026-11-11 10:23:20 +08:00
86 changed files with 6905 additions and 12517 deletions
-5
View File
@@ -70,11 +70,6 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
@echo "📦 Checking s3s footprint ratchet..."
./scripts/check_s3s_footprint.sh
.PHONY: gateway-stack-release-boundary-check
gateway-stack-release-boundary-check: ## Check no release build can enable the gateway-stack feature
@echo "🚧 Checking gateway-stack release boundary..."
./scripts/check_gateway_stack_release_boundary.sh
.PHONY: test-count-ratchet-check
test-count-ratchet-check: ## Check no workspace crate fell below its test floor or rose above its #[ignore] ceiling in the committed baseline
@echo "🧪 Checking per-crate test-count ratchet..."
+3 -3
View File
@@ -20,13 +20,13 @@ planning-docs-check: ## Check that no planning-type documents are committed
./scripts/check_no_planning_docs.sh
.PHONY: pre-commit
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check gateway-stack-release-boundary-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
@echo "✅ All pre-commit checks passed!"
.PHONY: pre-pr
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check gateway-stack-release-boundary-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
@echo "✅ All pre-PR checks passed!"
.PHONY: dev-check
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check gateway-stack-release-boundary-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
@echo "✅ Fast development checks passed!"
+2 -1
View File
@@ -61,8 +61,8 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/test_check_doc_paths.py
./scripts/check_embedded_secrets.sh --self-test
./scripts/check_s3s_footprint.sh --self-test
./scripts/check_gateway_stack_release_boundary.sh --self-test
./scripts/test_check_test_count_ratchet.sh
./scripts/test_guard_source_set.sh
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
@@ -94,6 +94,7 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_api_coverage.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_reference_parity.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
+2 -2
View File
@@ -137,7 +137,7 @@ filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair
setup = 'ecstore-base-stack'
[[profile.default.scripts]]
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
# These handler/store fixtures and the relocated-pool GET compose deep storage
@@ -455,7 +455,7 @@ filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair
setup = 'ecstore-base-stack'
[[profile.ci.scripts]]
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
[[profile.ci.scripts]]
+1 -1
View File
@@ -13,5 +13,5 @@
# header's rule clears (rustfs_s3_types's macro); fails closed
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
files=65
s3_error_lines=217
s3_error_lines=219
ecstore_files=0
-1
View File
@@ -25,7 +25,6 @@ rustfs-data-usage 58 0
rustfs-ecstore 7858 2
rustfs-extension-schema 23 0
rustfs-filemeta 361 0
rustfs-gateway-adapter 36 0
rustfs-heal 1128 3
rustfs-heal-contracts 8 0
rustfs-iam 415 0
-4
View File
@@ -97,10 +97,6 @@ runs:
shell: bash
run: ./scripts/check_s3s_footprint.sh
- name: Check gateway-stack release boundary
shell: bash
run: ./scripts/check_gateway_stack_release_boundary.sh
- name: Check per-crate test-count ratchet
shell: bash
run: ./scripts/check_test_count_ratchet.sh
+1 -50
View File
@@ -115,7 +115,7 @@ jobs:
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
echo "Base CI policy unavailable; running the full matrix."
fi
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=4$' "$GITHUB_OUTPUT"; then
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
echo "Base policy does not cover this workflow contract; retaining the full matrix."
fi
@@ -705,53 +705,6 @@ jobs:
retention-days: 3
if-no-files-found: warn
test-and-lint-gateway-stack:
name: Test and Lint (gateway-stack)
# The gateway S3 stack (rustfs/backlog#2734) sits behind the rustfs
# `gateway-stack` feature, which no other pull-request lane lints or tests.
# Unlike the dormant feature lanes above, this one runs for every code
# change: each operation slice puts its handler tests, its authorization
# census and its end-to-end refusal test behind the feature, and a lane that
# ran only on demand would let them rot behind a green pull request.
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
needs: [ quick-checks, classify-changes ]
# Self-hosted for build scale and cache ownership: a workspace-scale Cargo
# build that reads the ci-dev cache written on this image.
runs-on: sm-standard-4
timeout-minutes: 90
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-dev
cache-save-if: 'false'
install-build-packaging-tools: 'false'
# A feature that does not compile fails here; nothing below is skipped
# into a pass.
- name: Run clippy with gateway-stack
run: cargo clippy --locked -p rustfs --all-targets --features gateway-stack -- -D warnings
# The gateway stack's own tests: the assembly, the authorizer table and
# its census, the handler template, the migrated use cases, and the
# embedded server driven through the whole middleware stack.
- name: Run tests with gateway-stack
env:
# The same bounded link concurrency as the other feature lanes.
CARGO_BUILD_JOBS: "2"
run: |
cargo nextest run --profile ci --locked -p rustfs --features gateway-stack --no-tests fail \
--status-level all --final-status-level all \
-E 'test(/gateway_stack|gateway_handlers|server::stack::|list_buckets|legacy_stack_is_always_selectable/) | binary(embedded_gateway_stack_test)'
build-rustfs-debug-binary:
name: Build RustFS Debug Binary
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
@@ -1626,7 +1579,6 @@ jobs:
- fips-provider
- test-and-lint-rio-v2
- test-and-lint-protocols
- test-and-lint-gateway-stack
- build-rustfs-debug-binary
- uring-integration
- e2e-tests
@@ -1661,7 +1613,6 @@ jobs:
- core-contract
- test-and-lint-rio-v2
- test-and-lint-protocols
- test-and-lint-gateway-stack
- build-rustfs-debug-binary
- build-rustfs-debug-binary-rio-v2
- uring-integration
Generated
+2 -140
View File
@@ -4698,12 +4698,6 @@ version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
[[package]]
name = "futures-timer"
version = "3.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"
[[package]]
name = "futures-util"
version = "0.3.34"
@@ -7920,34 +7914,10 @@ version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"
dependencies = [
"phf_macros",
"phf_shared 0.14.0",
"serde",
]
[[package]]
name = "phf_generator"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100"
dependencies = [
"fastrand",
"phf_shared 0.14.0",
]
[[package]]
name = "phf_macros"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085"
dependencies = [
"phf_generator",
"phf_shared 0.14.0",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "phf_shared"
version = "0.12.1"
@@ -9639,8 +9609,6 @@ dependencies = [
"rustfs-ecstore",
"rustfs-extension-schema",
"rustfs-filemeta",
"rustfs-gateway",
"rustfs-gateway-adapter",
"rustfs-gateway-types",
"rustfs-heal",
"rustfs-heal-contracts",
@@ -10031,112 +9999,6 @@ dependencies = [
"xxhash-rust",
]
[[package]]
name = "rustfs-gateway"
version = "0.57.0"
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
dependencies = [
"arc-swap",
"bytes",
"form_urlencoded",
"futures-timer",
"futures-util",
"http 1.5.0",
"http-body",
"http-body-util",
"httparse",
"hyper",
"pin-project-lite",
"rustfs-gateway-core",
"rustfs-gateway-http",
"rustfs-gateway-macros",
"rustfs-gateway-sig",
"rustfs-gateway-stream",
"rustfs-gateway-types",
"rustfs-gateway-xml",
"sha2 0.11.1",
"tokio",
"tower",
"tracing",
"url",
]
[[package]]
name = "rustfs-gateway-adapter"
version = "1.0.5"
dependencies = [
"rustfs-credentials",
"rustfs-gateway",
"rustfs-iam",
"rustfs-policy",
"serde_json",
"time",
"tokio",
]
[[package]]
name = "rustfs-gateway-core"
version = "0.54.0"
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
dependencies = [
"bytes",
"http 1.5.0",
"md-5 0.11.0",
"percent-encoding",
"rustfs-gateway-http",
"rustfs-gateway-sig",
"rustfs-gateway-stream",
"rustfs-gateway-types",
"rustfs-gateway-xml",
"serde",
"serde_json",
"subtle",
"zeroize",
]
[[package]]
name = "rustfs-gateway-http"
version = "0.1.4"
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
dependencies = [
"hmac 0.13.0",
"http 1.5.0",
"httparse",
"memchr",
"phf 0.14.0",
"rustfs-gateway-stream",
"rustfs-gateway-types",
"sha2 0.11.1",
"smallvec",
"subtle",
"zeroize",
]
[[package]]
name = "rustfs-gateway-macros"
version = "0.1.4"
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.9",
]
[[package]]
name = "rustfs-gateway-sig"
version = "0.13.0"
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
dependencies = [
"hmac 0.13.0",
"http 1.5.0",
"rustfs-gateway-http",
"sha1 0.11.0",
"sha2 0.11.1",
"smallvec",
"subtle",
"zeroize",
]
[[package]]
name = "rustfs-gateway-stream"
version = "0.2.0"
@@ -12741,10 +12603,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.3.4",
"getrandom 0.4.3",
"once_cell",
"rustix",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
-6
View File
@@ -26,7 +26,6 @@ members = [
"crates/ecstore", # Erasure coding storage implementation
"crates/e2e_test", # End-to-end test suite
"crates/filemeta", # File metadata management
"crates/gateway-adapter", # RustFS implementations of the gateway extension traits
"crates/heal", # Erasure set and object healing
"crates/heal-contracts", # Heal request/response channel contracts
"crates/iam", # Identity and Access Management
@@ -94,7 +93,6 @@ redundant_clone = "warn"
[workspace.dependencies]
# RustFS Internal Crates
rustfs = { path = "./rustfs", version = "1.0.5" }
rustfs-gateway-adapter = { path = "crates/gateway-adapter", version = "1.0.5" }
rustfs-heal = { path = "crates/heal", version = "1.0.5" }
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.5" }
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.5" }
@@ -151,10 +149,6 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
# the s3s-facing edge of the rustfs binary.
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.32.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.2.0", default-features = false }
# The gateway framework itself, consumed for its extension traits by the ring-2
# adapter. `default-features = false` keeps the `server` feature, and with it the
# server crate, out of the build.
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.57.0", default-features = false }
# Async Runtime and Networking
async-channel = "2.5.0"
-46
View File
@@ -1,46 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package]
name = "rustfs-gateway-adapter"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
homepage.workspace = true
description = "RustFS implementations of the rustfs-gateway extension traits (credentials and policy facts over IAM)."
keywords = ["rustfs", "gateway", "s3", "iam"]
categories = ["web-programming", "authentication"]
authors.workspace = true
publish = false
[package.metadata.gateway]
# Documentary only: this crate is a ring-2 consumer of the gateway (it depends on
# RustFS business crates). The RustFS repository does not run the gateway's ring guard.
ring = 2
[lints]
workspace = true
[dependencies]
rustfs-gateway = { workspace = true }
rustfs-credentials = { workspace = true }
rustfs-iam = { workspace = true }
rustfs-policy = { workspace = true }
serde_json = { workspace = true }
[dev-dependencies]
time = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt"] }
-446
View File
@@ -1,446 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The gateway's [`CredentialProvider`], answered by RustFS IAM.
//!
//! Responsible for: resolving one access key id to the secret the gateway verifies a signature
//! with, and classifying every other outcome as either a business result
//! ([`CredentialLookup::NotFound`], eligible for the gateway's negative cache) or an operational
//! fault ([`ProviderError`], counted and never cached).
//! NOT responsible for: authorization of any kind, session-token claim validation, or deciding
//! what the client is answered with. The gateway answers every unusable credential identically.
//!
//! # Secret handling
//!
//! Secrets travel only inside the gateway's [`Credentials`], whose `Debug` redacts them. No type
//! in this module derives or implements `Debug`, and no error produced here carries text: the
//! IAM error is classified and dropped. A secret that fails the crate's effective-length rule
//! (`crate::MIN_SECRET_KEY_LEN` bytes once trailing NULs are removed) is never handed to the
//! gateway, whatever the lookup says about its validity.
use std::sync::Arc;
use rustfs_gateway::{BoxFuture, CredentialLookup, CredentialProvider, Credentials, ProviderError, SessionBinding};
use rustfs_iam::error::Error as IamError;
use rustfs_iam::store::Store;
use rustfs_iam::sys::IamSys;
use rustfs_policy::auth::{ACCOUNT_OFF, UserIdentity};
use crate::root::{RootSource, is_usable_secret};
/// The opaque issuer recorded in the [`rustfs_gateway::SessionBinding`] of every RustFS STS
/// credential.
pub const STS_SESSION_ISSUER: &str = "rustfs-sts";
/// Access keys with this prefix belong to Keystone token authentication, which never verifies an
/// AWS signature.
const KEYSTONE_ACCESS_KEY_PREFIX: &str = "keystone:";
/// The narrow slice of IAM that a credential lookup needs.
///
/// Exists so the adapter can be tested against a fake: IAM exports no in-memory backend.
/// Implemented for [`IamSys`], whose `check_key` loads a user from storage on a cache miss.
pub trait IamKeyLookup: Send + Sync + 'static {
/// Whether the IAM store has finished loading.
fn is_ready(&self) -> bool;
/// Resolves an access key, with [`IamSys::check_key`]'s contract: `Ok((None, _))` for an
/// unknown key, `Ok((Some(identity), valid))` otherwise, where `valid` is false for a
/// disabled, expired, or malformed credential.
fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>>;
}
impl<T: Store> IamKeyLookup for IamSys<T> {
fn is_ready(&self) -> bool {
IamSys::is_ready(self)
}
fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
Box::pin(IamSys::check_key(self, access_key))
}
}
/// A [`CredentialProvider`] over RustFS IAM and the live root credentials.
///
/// Built only by [`crate::IamAdapters::new`], so it always reads the same [`RootSource`] as the
/// policy source it was assembled with.
pub struct IamCredentials {
iam: Arc<dyn IamKeyLookup>,
root: Arc<dyn RootSource>,
}
impl IamCredentials {
pub(crate) fn new(iam: Arc<dyn IamKeyLookup>, root: Arc<dyn RootSource>) -> Self {
Self { iam, root }
}
async fn resolve(&self, access_key: &str) -> Result<CredentialLookup, ProviderError> {
// Deliberate change from the legacy s3s path (`rustfs/src/auth.rs` `get_secret_key`),
// which answered a `keystone:` key with an empty secret. Keystone authenticates by token,
// so there is no AWS secret to verify a signature against: the key is unknown here.
if access_key.starts_with(KEYSTONE_ACCESS_KEY_PREFIX) {
return Ok(CredentialLookup::NotFound);
}
// Legacy answers every key with an internal error while the root credentials are not
// initialized; without them no key can be classified as root or not.
let Some(root) = self.root.current() else {
return Err(ProviderError::Unavailable);
};
// Byte-exact, as the trait requires: the access key id is signed material.
if root.access_key() == access_key {
return Ok(CredentialLookup::Found(root.credentials().clone_credentials()));
}
if !self.iam.is_ready() {
return Err(ProviderError::Unavailable);
}
match self.iam.check_key(access_key).await {
// IAM refuses such a secret through `valid`, but this provider does not rely on every
// lookup doing so: a short secret is never usable, so it is not a credential at all.
Ok((Some(identity), _)) if !is_usable_secret(identity.credentials.secret_key.as_bytes()) => {
Ok(CredentialLookup::NotFound)
}
Ok((Some(identity), valid)) => to_gateway_credentials(access_key, &identity, valid).map(CredentialLookup::Found),
Ok((None, _)) => Ok(CredentialLookup::NotFound),
Err(error) => classify_iam_error(&error),
}
}
}
/// Classifies an IAM failure without keeping any of its text.
fn classify_iam_error(error: &IamError) -> Result<CredentialLookup, ProviderError> {
match error {
IamError::NoSuchUser(_)
| IamError::NoSuchServiceAccount(_)
| IamError::NoSuchTempAccount(_)
| IamError::NoSuchAccount(_) => Ok(CredentialLookup::NotFound),
IamError::IamSysNotInitialized => Err(ProviderError::Unavailable),
_ => Err(ProviderError::Backend),
}
}
/// Builds the gateway credential for one IAM identity. No authorization happens here: a
/// disabled or invalid principal still resolves, marked disabled, and the gateway refuses it
/// exactly as it refuses an unknown key.
fn to_gateway_credentials(access_key: &str, identity: &UserIdentity, valid: bool) -> Result<Credentials, ProviderError> {
let cred = &identity.credentials;
let mut credentials = Credentials::new(access_key, cred.secret_key.as_bytes()).map_err(|_| ProviderError::Backend)?;
// A service account carries a claims token internally, but its clients never present it, so
// it authenticates as a long-term credential (as `check_session_token` treats it in legacy).
if !cred.session_token.is_empty() && !cred.is_service_account() {
// A temporary credential without an expiry cannot become a gateway session, which always
// ends. Refusing it as a fault is the fail-closed answer; a never-ending session is not.
let expiration = cred.expiration.ok_or(ProviderError::Backend)?;
let binding = SessionBinding::new(STS_SESSION_ISSUER, expiration.unix_timestamp()).map_err(|_| ProviderError::Backend)?;
credentials = credentials
.with_session(&cred.session_token, binding)
.map_err(|_| ProviderError::Backend)?;
}
if !valid || cred.status == ACCOUNT_OFF {
credentials = credentials.disable();
}
Ok(credentials)
}
impl CredentialProvider for IamCredentials {
fn lookup<'a>(&'a self, access_key_id: &'a str) -> BoxFuture<'a, Result<CredentialLookup, ProviderError>> {
Box::pin(self.resolve(access_key_id))
}
}
#[cfg(test)]
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
mod tests {
use super::*;
use crate::root::{RootCell, RootCredentials};
use rustfs_credentials::Credentials as IamCredential;
use rustfs_gateway::RequestNow;
use std::sync::atomic::{AtomicUsize, Ordering};
use time::OffsetDateTime;
const ACCESS_KEY: &str = "AKIDUSER0001";
const SECRET: &str = "user-secret-key-0001";
type Answer = Box<dyn Fn() -> Result<(Option<UserIdentity>, bool), IamError> + Send + Sync>;
struct FakeIam {
ready: bool,
answer: Answer,
calls: AtomicUsize,
}
impl IamKeyLookup for FakeIam {
fn is_ready(&self) -> bool {
self.ready
}
fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
self.calls.fetch_add(1, Ordering::SeqCst);
let answer = (self.answer)();
Box::pin(async move { answer })
}
}
fn fake(
ready: bool,
answer: impl Fn() -> Result<(Option<UserIdentity>, bool), IamError> + Send + Sync + 'static,
) -> Arc<FakeIam> {
Arc::new(FakeIam {
ready,
answer: Box::new(answer),
calls: AtomicUsize::new(0),
})
}
fn user(status: &str) -> IamCredential {
IamCredential {
access_key: ACCESS_KEY.to_owned(),
secret_key: SECRET.to_owned(),
status: status.to_owned(),
..Default::default()
}
}
fn found(cred: IamCredential, valid: bool) -> Result<(Option<UserIdentity>, bool), IamError> {
Ok((Some(UserIdentity::new(cred)), valid))
}
fn root_cell() -> Arc<RootCell> {
let root = RootCredentials::new("rootadmin", "root-secret-key", "sts-signing-key").expect("valid root credentials");
Arc::new(RootCell::new(Some(root)))
}
async fn lookup(iam: Arc<FakeIam>, access_key: &str) -> Result<CredentialLookup, ProviderError> {
IamCredentials::new(iam, root_cell()).lookup(access_key).await
}
fn expect_found(result: Result<CredentialLookup, ProviderError>) -> Credentials {
match result {
Ok(CredentialLookup::Found(credentials)) => credentials,
Ok(CredentialLookup::NotFound) => panic!("expected Found, got NotFound"),
Err(error) => panic!("expected Found, got {error:?}"),
}
}
fn is_not_found(result: &Result<CredentialLookup, ProviderError>) -> bool {
matches!(result, Ok(CredentialLookup::NotFound))
}
/// Positive — an enabled user resolves to exactly its secret, as a long-term credential.
#[tokio::test]
async fn credentials_found_user_resolves_to_its_secret() {
let iam = fake(true, || found(user("on"), true));
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
assert_eq!(credentials.secret().expose(), SECRET.as_bytes());
assert_eq!(credentials.identity().access_key_id(), ACCESS_KEY);
assert!(!credentials.is_disabled());
assert!(credentials.session_binding().is_none());
}
/// Negative — a key IAM does not know is a business result, not a fault.
#[tokio::test]
async fn credentials_unknown_key_is_not_found() {
let iam = fake(true, || Ok((None, false)));
assert!(is_not_found(&lookup(iam, "AKIDUNKNOWN").await));
}
/// Negative — every "no such principal" error IAM can surface is `NotFound`. Answering these
/// as `Backend` would keep them out of the negative cache and count every guessed key as a
/// backend fault.
#[tokio::test]
async fn credentials_no_such_principal_errors_are_not_found() {
let errors: [fn() -> IamError; 4] = [
|| IamError::NoSuchUser(ACCESS_KEY.to_owned()),
|| IamError::NoSuchServiceAccount(ACCESS_KEY.to_owned()),
|| IamError::NoSuchTempAccount(ACCESS_KEY.to_owned()),
|| IamError::NoSuchAccount(ACCESS_KEY.to_owned()),
];
for error in errors {
let iam = fake(true, move || Err(error()));
let result = lookup(iam, ACCESS_KEY).await;
assert!(is_not_found(&result), "{:?} must be NotFound", error());
}
}
/// Negative — any other IAM failure is a backend fault, never `NotFound` and never `Found`.
#[tokio::test]
async fn credentials_backend_error_is_backend() {
let iam = fake(true, || Err(IamError::StringError("storage read failed".to_owned())));
assert!(matches!(lookup(iam, ACCESS_KEY).await, Err(ProviderError::Backend)));
}
/// Negative — a `keystone:` key never reaches IAM and never resolves to a secret.
///
/// Deliberate change from the legacy s3s path, which answered such a key with an empty
/// secret: Keystone authenticates by token, so there is no AWS secret to verify against.
#[tokio::test]
async fn credentials_keystone_prefix_is_not_found_without_iam() {
let iam = fake(true, || found(user("on"), true));
let result = lookup(iam.clone(), "keystone:project-user").await;
assert!(is_not_found(&result));
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
}
/// Negative — a switched-off or otherwise invalid user resolves, but disabled, and the same
/// user enabled does not.
#[tokio::test]
async fn credentials_disabled_user_is_disabled() {
let off = expect_found(lookup(fake(true, || found(user("off"), false)), ACCESS_KEY).await);
assert!(off.is_disabled());
assert!(off.admit(None, RequestNow::from_unix_seconds(0)).is_err());
let invalid = expect_found(lookup(fake(true, || found(user("on"), false)), ACCESS_KEY).await);
assert!(invalid.is_disabled());
let on = expect_found(lookup(fake(true, || found(user("on"), true)), ACCESS_KEY).await);
assert!(!on.is_disabled());
}
/// Positive — an STS credential carries its token and its lifetime as one session binding.
#[tokio::test]
async fn credentials_sts_carries_session_binding() {
let expires_at = 4_000_000_000_i64;
let iam = fake(true, move || {
let mut cred = user("on");
cred.session_token = "sts-session-token".to_owned();
cred.parent_user = "parent".to_owned();
cred.expiration = Some(OffsetDateTime::from_unix_timestamp(expires_at).expect("valid timestamp"));
found(cred, true)
});
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
let binding = credentials.session_binding().expect("an STS credential has a session");
assert_eq!(binding.expires_at_unix_seconds(), expires_at);
assert_eq!(binding.issuer(), STS_SESSION_ISSUER);
let now = RequestNow::from_unix_seconds(expires_at - 1);
assert!(credentials.admit(Some(b"sts-session-token"), now).is_ok());
assert!(credentials.admit(None, now).is_err());
}
/// Negative — an STS credential with no expiry cannot be expressed as a session the gateway
/// would ever expire, so it is a backend fault rather than a session that never ends.
#[tokio::test]
async fn credentials_sts_without_expiry_is_backend() {
let iam = fake(true, || {
let mut cred = user("on");
cred.session_token = "sts-session-token".to_owned();
found(cred, true)
});
assert!(matches!(lookup(iam, ACCESS_KEY).await, Err(ProviderError::Backend)));
}
/// Positive — a service account carries a token internally but authenticates as a long-term
/// credential: clients never present that token.
#[tokio::test]
async fn credentials_service_account_is_long_term() {
let iam = fake(true, || {
let mut cred = user("on");
cred.session_token = "service-account-claims-token".to_owned();
cred.parent_user = "parent".to_owned();
cred.claims = Some([(rustfs_credentials::IAM_POLICY_CLAIM_NAME_SA.to_owned(), "inherited-policy".into())].into());
found(cred, true)
});
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
assert!(credentials.session_binding().is_none());
assert!(credentials.admit(None, RequestNow::from_unix_seconds(0)).is_ok());
}
/// Negative — before IAM is ready a lookup is `Unavailable`, not `NotFound`: a key must not
/// be negatively cached because the store had not loaded yet.
#[tokio::test]
async fn credentials_iam_not_ready_is_unavailable() {
let iam = fake(false, || found(user("on"), true));
assert!(matches!(lookup(iam.clone(), ACCESS_KEY).await, Err(ProviderError::Unavailable)));
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
let uninitialized = fake(true, || Err(IamError::IamSysNotInitialized));
assert!(matches!(lookup(uninitialized, ACCESS_KEY).await, Err(ProviderError::Unavailable)));
}
/// Positive — the injected root credentials resolve without IAM, even before it is ready, and
/// nothing else is answered with the root secret.
#[tokio::test]
async fn credentials_root_resolves_without_iam() {
let iam = fake(false, || found(user("on"), true));
let provider = IamCredentials::new(iam.clone(), root_cell());
let root = expect_found(provider.lookup("rootadmin").await);
assert_eq!(root.secret().expose(), b"root-secret-key");
assert!(matches!(provider.lookup("ROOTADMIN").await, Err(ProviderError::Unavailable)));
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
}
/// Negative — a lookup that calls an identity valid but holds an empty or short secret is not
/// trusted: such a secret is not a credential, so the key is `NotFound`, never `Found`.
#[tokio::test]
async fn credentials_short_secret_is_never_found() {
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
let iam = fake(true, move || {
let mut cred = user("on");
cred.secret_key = weak.to_owned();
found(cred, true)
});
let result = lookup(iam, ACCESS_KEY).await;
assert!(is_not_found(&result), "{weak:?} must be NotFound");
}
for usable in ["12345678", "abc\0defg"] {
let iam = fake(true, move || {
let mut cred = user("on");
cred.secret_key = usable.to_owned();
found(cred, true)
});
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
assert_eq!(credentials.secret().expose(), usable.as_bytes());
}
}
/// Negative — a switched-off account is disabled even when the lookup calls it valid.
#[tokio::test]
async fn credentials_off_status_alone_disables() {
let off = expect_found(lookup(fake(true, || found(user("off"), true)), ACCESS_KEY).await);
assert!(off.is_disabled());
assert!(off.admit(None, RequestNow::from_unix_seconds(0)).is_err());
}
/// Negative — an STS credential admits only its own session's token: the token of another
/// session, or none, is refused.
#[tokio::test]
async fn credentials_sts_refuses_another_sessions_token() {
let expires_at = 4_000_000_000_i64;
let iam = fake(true, move || {
let mut cred = user("on");
cred.session_token = "session-token-a".to_owned();
cred.parent_user = "parent".to_owned();
cred.expiration = Some(OffsetDateTime::from_unix_timestamp(expires_at).expect("valid timestamp"));
found(cred, true)
});
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
let now = RequestNow::from_unix_seconds(expires_at - 1);
assert!(credentials.admit(Some(b"session-token-b"), now).is_err());
assert!(credentials.admit(Some(b"session-token-a"), now).is_ok());
}
/// Negative — without root credentials no key can be classified, so every lookup is
/// `Unavailable` and IAM is not asked.
#[tokio::test]
async fn credentials_without_root_are_unavailable() {
let iam = fake(true, || found(user("on"), true));
let provider = IamCredentials::new(iam.clone(), Arc::new(RootCell::new(None)));
assert!(matches!(provider.lookup(ACCESS_KEY).await, Err(ProviderError::Unavailable)));
assert!(matches!(provider.lookup("rootadmin").await, Err(ProviderError::Unavailable)));
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
}
}
-99
View File
@@ -1,99 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! RustFS implementations of the `rustfs-gateway` extension traits.
//!
//! This is the gateway's ring 2: it may depend on RustFS business crates, and nothing in the
//! gateway depends on it. Each module adapts one extension point to the RustFS subsystem that
//! owns the answer.
pub mod credentials;
pub mod policy;
pub mod root;
use std::sync::Arc;
use rustfs_gateway::{Authenticator, CredentialProvider, ServiceBuilder};
pub use credentials::{IamCredentials, IamKeyLookup, STS_SESSION_ISSUER};
pub use policy::{IamPolicySource, IamPrincipalLookup, PrincipalKind, RustfsPrincipal};
pub use root::{MIN_SECRET_KEY_LEN, RootCredentials, RootCredentialsError, RootSource};
/// The IAM-backed credential provider and policy source, assembled together.
///
/// Both halves read the same IAM handle and the same live [`RootSource`], so a key cannot be root
/// to one and an ordinary user to the other. Neither half can be taken out on its own: the fields
/// are private, and [`IamAdapters::install`] consumes the pair and hands both to one builder.
///
/// Mixing the halves of two assemblies does not compile:
///
/// ```compile_fail
/// fn mix(a: rustfs_gateway_adapter::IamAdapters, b: rustfs_gateway_adapter::IamAdapters) {
/// let _mixed = (a.credentials, b.policy);
/// }
/// ```
///
/// Nor does building a half outside the pair:
///
/// ```compile_fail
/// use std::sync::Arc;
/// use rustfs_gateway_adapter::{IamPolicySource, IamPrincipalLookup, RootSource};
/// fn build(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) {
/// let _alone = IamPolicySource::new(iam, root);
/// }
/// ```
///
/// Installing the pair is the one supported path:
///
/// ```no_run
/// use std::sync::Arc;
/// use rustfs_gateway::{RegionSet, ServiceBuilder, SigV4Authenticator};
/// use rustfs_gateway_adapter::{IamAdapters, IamPrincipalLookup, RootSource};
/// fn assemble(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>, regions: RegionSet) -> ServiceBuilder {
/// IamAdapters::new(iam, root).install(ServiceBuilder::new(), |provider| SigV4Authenticator::new(provider, regions))
/// }
/// ```
pub struct IamAdapters {
credentials: IamCredentials,
policy: IamPolicySource,
}
impl IamAdapters {
/// Assembles both adapters over one IAM handle and one root source. The binary passes a
/// source that reads its live root credentials on every call (see [`RootSource`]).
#[must_use]
pub fn new(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) -> Self {
Self {
credentials: IamCredentials::new(iam.clone(), root.clone()),
policy: IamPolicySource::new(iam, root),
}
}
/// Installs both halves into `builder`: the credential provider through the authenticator
/// `authenticator` builds over it (for example `SigV4Authenticator::new(provider, regions)`
/// with the RustFS profile applied), and the policy source.
///
/// Consumes the pair and gives both halves to this one builder. The `authenticator` closure
/// receives the provider handle and may keep a clone of it; nothing here prevents that, so
/// the closure must not hand it to a second builder.
#[must_use]
pub fn install<A: Authenticator>(
self,
builder: ServiceBuilder,
authenticator: impl FnOnce(Arc<dyn CredentialProvider>) -> A,
) -> ServiceBuilder {
let authenticator = authenticator(Arc::new(self.credentials));
builder.authenticator(authenticator).policy_source(self.policy)
}
}
-779
View File
@@ -1,779 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The gateway's [`PolicySource`], answered by RustFS IAM.
//!
//! Responsible for: the one reading of the caller's IAM facts a request is judged against — which
//! principal kind it is, the user it acts for, its groups, its token claims, and whether it owns
//! the deployment — packed as a [`RustfsPrincipal`] into the gateway's [`PolicySnapshot`].
//! NOT responsible for: evaluating any policy, choosing which policy documents apply (that stays
//! in `IamSys::is_allowed`, which owns the STS, OIDC, session-policy and service-account rules), or
//! verifying the signature or the session token (the gateway did both before calling this).
//! Upstream: the gateway's verified [`Identity`]. Downstream: the RustFS authorizer, which reads
//! the payload back with `PolicySnapshot::get::<RustfsPrincipal>()`.
//!
//! # Failure classes
//!
//! Every refusal is a [`PolicyError`] with a constant label and nothing from IAM or the request:
//! IAM not ready, not initialized, or failing, or root credentials not initialized, is
//! [`PolicyError::unavailable`]; a principal IAM no longer knows is [`UNKNOWN_PRINCIPAL`]; a
//! disabled one, or one whose secret fails the effective-length rule, is
//! [`DISABLED_PRINCIPAL`]; an identity
//! whose credential kind disagrees with IAM's record is [`SESSION_MISMATCH`]; a token whose claims
//! cannot be read is [`PolicyError::malformed`]. None of them is ever an empty snapshot: an empty
//! snapshot is what an anonymous caller gets, and answering a fault with it would judge an
//! authenticated caller as anonymous.
//!
//! # Secret handling
//!
//! [`RustfsPrincipal`] holds no secret key and no session token, and implements no `Debug`, so a
//! snapshot formatted into a log cannot print its claims either. The root credentials and the STS
//! token-signing key are read per request from the shared [`RootSource`] and never stored here.
use std::collections::HashMap;
use std::sync::Arc;
use rustfs_credentials::Credentials as IamCredential;
use rustfs_gateway::{BoxFuture, Identity, PolicyError, PolicySnapshot, PolicySource};
use rustfs_iam::error::Error as IamError;
use rustfs_iam::store::Store;
use rustfs_iam::sys::{
IamSys, SESSION_POLICY_NAME, get_claims_from_token_with_secret, get_claims_from_token_with_secret_allow_missing_exp,
is_rustfs_oidc_claims,
};
use rustfs_policy::auth::ACCOUNT_OFF;
use serde_json::Value;
use crate::credentials::IamKeyLookup;
use crate::root::{RootCredentials, RootSource, is_usable_secret};
/// [`PolicyError`] label: IAM does not know the authenticated principal (any longer).
pub const UNKNOWN_PRINCIPAL: &str = "unknown-principal";
/// [`PolicyError`] label: IAM holds the principal disabled, expired, or otherwise invalid.
pub const DISABLED_PRINCIPAL: &str = "disabled-principal";
/// [`PolicyError`] label: the identity's credential kind (session or long-term) disagrees with
/// what IAM holds for its access key.
pub const SESSION_MISMATCH: &str = "session-mismatch";
/// The narrow slice of IAM a policy reading needs, on top of [`IamKeyLookup`].
///
/// Implemented for [`IamSys`]; exists so the source can be tested against a fake.
pub trait IamPrincipalLookup: IamKeyLookup {
/// The built-in IAM groups `user` belongs to now, as `IamSys::user_group_memberships`
/// answers. These may include groups granted after a credential was issued.
fn group_memberships(&self, user: &str) -> Vec<String>;
}
impl<T: Store> IamPrincipalLookup for IamSys<T> {
fn group_memberships(&self, user: &str) -> Vec<String> {
IamSys::user_group_memberships(self, user)
}
}
/// Which kind of credential the principal authenticated with.
#[derive(Clone, Copy, PartialEq, Eq)]
pub enum PrincipalKind {
/// The deployment's root credentials.
Root,
/// A long-term IAM user.
User,
/// A service account, acting for its parent user.
ServiceAccount,
/// An STS temporary credential, acting for its parent user.
Temporary,
}
/// The caller's IAM facts: the payload of every [`PolicySnapshot`] [`IamPolicySource`] produces.
///
/// Carries no secret key and no session token, and deliberately implements no `Debug`.
pub struct RustfsPrincipal {
kind: PrincipalKind,
account: String,
username: String,
parent_user: String,
credential_groups: Option<Vec<String>>,
group_memberships: Vec<String>,
claims: Option<HashMap<String, Value>>,
is_owner: bool,
}
impl RustfsPrincipal {
/// The kind of credential the caller authenticated with.
#[must_use]
pub fn kind(&self) -> PrincipalKind {
self.kind
}
/// The access key the caller signed with (IAM `Args::account`).
#[must_use]
pub fn account(&self) -> &str {
&self.account
}
/// The user the caller acts as: the parent user for an STS credential or a service account,
/// the access key otherwise.
#[must_use]
pub fn username(&self) -> &str {
&self.username
}
/// The parent user IAM records for the credential; empty for root and long-term users.
#[must_use]
pub fn parent_user(&self) -> &str {
&self.parent_user
}
/// The groups recorded on the credential itself (IAM `Args::groups`).
#[must_use]
pub fn credential_groups(&self) -> &Option<Vec<String>> {
&self.credential_groups
}
/// The groups IAM currently lists for [`Self::username`], read once with the snapshot.
#[must_use]
pub fn group_memberships(&self) -> &[String] {
&self.group_memberships
}
/// The claims the credential's token carries; `None` when it has none.
#[must_use]
pub fn claims(&self) -> Option<&HashMap<String, Value>> {
self.claims.as_ref()
}
/// Whether the caller owns the deployment (IAM `Args::is_owner`): root, or a credential of
/// root that no session policy or external OIDC identity narrows.
#[must_use]
pub fn is_owner(&self) -> bool {
self.is_owner
}
}
/// A [`PolicySource`] over RustFS IAM and the live root credentials.
///
/// Built only by [`crate::IamAdapters::new`], so it always reads the same [`RootSource`] as the
/// credential provider it was assembled with.
pub struct IamPolicySource {
iam: Arc<dyn IamPrincipalLookup>,
root: Arc<dyn RootSource>,
}
impl IamPolicySource {
pub(crate) fn new(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) -> Self {
Self { iam, root }
}
async fn read(&self, identity: Option<&Identity>) -> Result<PolicySnapshot, PolicyError> {
let Some(identity) = identity else {
return Ok(PolicySnapshot::empty());
};
let principal = self.principal(identity).await?;
Ok(PolicySnapshot::of(Arc::new(principal)))
}
async fn principal(&self, identity: &Identity) -> Result<RustfsPrincipal, PolicyError> {
let access_key = identity.access_key_id();
let has_session = identity.session().is_some();
// Without root credentials neither ownership nor STS claims can be decided.
let Some(root) = self.root.current() else {
return Err(PolicyError::unavailable());
};
let root_access_key = root.access_key();
if access_key == root_access_key {
// Root has no session token, so a session identity for its key is not one IAM issued.
if has_session {
return Err(PolicyError::because(SESSION_MISMATCH));
}
return Ok(RustfsPrincipal {
kind: PrincipalKind::Root,
account: access_key.to_owned(),
username: access_key.to_owned(),
parent_user: String::new(),
credential_groups: None,
group_memberships: Vec::new(),
claims: None,
is_owner: true,
});
}
if !self.iam.is_ready() {
return Err(PolicyError::unavailable());
}
let cred = match self.iam.check_key(access_key).await {
Ok((Some(identity), true)) => identity.credentials,
Ok((Some(_), false)) => return Err(PolicyError::because(DISABLED_PRINCIPAL)),
Ok((None, _)) => return Err(PolicyError::because(UNKNOWN_PRINCIPAL)),
Err(error) => return Err(classify_iam_error(&error)),
};
if cred.status == ACCOUNT_OFF || !is_usable_secret(cred.secret_key.as_bytes()) {
return Err(PolicyError::because(DISABLED_PRINCIPAL));
}
let kind = if cred.is_service_account() {
PrincipalKind::ServiceAccount
} else if !cred.session_token.is_empty() {
PrincipalKind::Temporary
} else {
PrincipalKind::User
};
// The gateway admitted this identity against the credential IAM returned, so the kinds
// agree unless IAM's record changed in between; refuse rather than judge a stale reading.
if has_session != (kind == PrincipalKind::Temporary) {
return Err(PolicyError::because(SESSION_MISMATCH));
}
let claims = read_claims(kind, &cred, &root)?;
let username = match kind {
PrincipalKind::Temporary | PrincipalKind::ServiceAccount => cred.parent_user.clone(),
PrincipalKind::Root | PrincipalKind::User => cred.access_key.clone(),
};
let membership_subject = if username.is_empty() { access_key } else { username.as_str() };
let group_memberships = self.iam.group_memberships(membership_subject);
let is_owner = acts_for_root(&cred, claims.as_ref(), root_access_key);
Ok(RustfsPrincipal {
kind,
account: access_key.to_owned(),
username,
parent_user: cred.parent_user,
credential_groups: cred.groups,
group_memberships,
claims,
is_owner,
})
}
}
/// The claims of the credential's own token. The gateway admitted the presented token only when it
/// equals the stored one, so reading the stored token reads the presented one.
fn read_claims(
kind: PrincipalKind,
cred: &IamCredential,
root: &RootCredentials,
) -> Result<Option<HashMap<String, Value>>, PolicyError> {
// As in legacy `check_claims_from_token_with_context`: no token, no claims.
if cred.session_token.is_empty() {
return Ok(None);
}
let claims = match kind {
PrincipalKind::ServiceAccount => {
get_claims_from_token_with_secret_allow_missing_exp(&cred.session_token, &cred.secret_key)
.map_err(|_| PolicyError::malformed())?
}
PrincipalKind::Temporary => get_claims_from_token_with_secret(&cred.session_token, root.sts_signing_key())
.map_err(|_| PolicyError::malformed())?,
PrincipalKind::Root | PrincipalKind::User => return Ok(None),
};
Ok(if claims.is_empty() { None } else { Some(claims) })
}
/// Classifies an IAM failure without keeping any of its text.
fn classify_iam_error(error: &IamError) -> PolicyError {
match error {
IamError::NoSuchUser(_)
| IamError::NoSuchServiceAccount(_)
| IamError::NoSuchTempAccount(_)
| IamError::NoSuchAccount(_) => PolicyError::because(UNKNOWN_PRINCIPAL),
_ => PolicyError::unavailable(),
}
}
/// Root ownership as the legacy edge decides it (`rustfs/src/auth.rs` `has_root_access`): the
/// credential is root's or acts for root, and neither a session policy nor an external OIDC
/// identity narrows it.
fn acts_for_root(cred: &IamCredential, claims: Option<&HashMap<String, Value>>, root_access_key: &str) -> bool {
(cred.access_key == root_access_key || cred.parent_user == root_access_key)
&& !claims.is_some_and(|claims| claims.contains_key(SESSION_POLICY_NAME) || is_rustfs_oidc_claims(claims))
}
impl PolicySource for IamPolicySource {
fn snapshot<'a>(&'a self, identity: Option<&'a Identity>) -> BoxFuture<'a, Result<PolicySnapshot, PolicyError>> {
Box::pin(self.read(identity))
}
}
#[cfg(test)]
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
mod tests {
use super::*;
use crate::root::RootCell;
use rustfs_credentials::{Credentials as IamCredential, IAM_POLICY_CLAIM_NAME_SA};
use rustfs_gateway::SessionBinding;
use rustfs_iam::error::Error as IamError;
use rustfs_iam::sys::SESSION_POLICY_NAME;
use rustfs_iam::utils::generate_jwt;
use rustfs_policy::auth::UserIdentity;
use std::sync::atomic::{AtomicUsize, Ordering};
use time::OffsetDateTime;
const ROOT_ACCESS_KEY: &str = "rootadmin";
const ROOT_SECRET: &str = "root-secret-key-0001";
/// Deliberately not the root secret, so every STS test proves which key claims are read with.
const SIGNING_KEY: &str = "sts-signing-key-0001";
const ACCESS_KEY: &str = "AKIDUSER0001";
const SECRET: &str = "user-secret-key-0001";
const PARENT: &str = "alice";
const EXPIRES_AT: i64 = 4_000_000_000;
type KeyAnswer = Result<(Option<UserIdentity>, bool), IamError>;
type Answer = Box<dyn Fn() -> KeyAnswer + Send + Sync>;
struct FakeIam {
ready: bool,
answer: Answer,
calls: AtomicUsize,
}
impl IamKeyLookup for FakeIam {
fn is_ready(&self) -> bool {
self.ready
}
fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, KeyAnswer> {
self.calls.fetch_add(1, Ordering::SeqCst);
let answer = (self.answer)();
Box::pin(async move { answer })
}
}
impl IamPrincipalLookup for FakeIam {
/// Distinct groups per name, so a test can tell which name the source asked about.
fn group_memberships(&self, user: &str) -> Vec<String> {
match user {
PARENT => vec!["parent-group".to_owned()],
ACCESS_KEY => vec!["key-group".to_owned()],
ROOT_ACCESS_KEY => vec!["root-group".to_owned()],
_ => Vec::new(),
}
}
}
fn fake(ready: bool, answer: impl Fn() -> KeyAnswer + Send + Sync + 'static) -> Arc<FakeIam> {
Arc::new(FakeIam {
ready,
answer: Box::new(answer),
calls: AtomicUsize::new(0),
})
}
fn root() -> RootCredentials {
RootCredentials::new(ROOT_ACCESS_KEY, ROOT_SECRET, SIGNING_KEY).expect("valid root credentials")
}
fn source(iam: Arc<FakeIam>) -> IamPolicySource {
IamPolicySource::new(iam, Arc::new(RootCell::new(Some(root()))))
}
fn user() -> IamCredential {
IamCredential {
access_key: ACCESS_KEY.to_owned(),
secret_key: SECRET.to_owned(),
status: "on".to_owned(),
groups: Some(vec!["cred-group".to_owned()]),
..Default::default()
}
}
fn sts_token(claims: &[(&str, Value)], secret: &str) -> String {
let mut map: HashMap<String, Value> = claims.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect();
map.insert("exp".to_owned(), Value::from(EXPIRES_AT));
generate_jwt(&map, secret).expect("a token")
}
fn sts(parent: &str, token: String) -> IamCredential {
IamCredential {
session_token: token,
parent_user: parent.to_owned(),
expiration: Some(OffsetDateTime::from_unix_timestamp(EXPIRES_AT).expect("valid timestamp")),
..user()
}
}
fn found(cred: IamCredential) -> KeyAnswer {
Ok((Some(UserIdentity::new(cred)), true))
}
fn long_term(access_key: &str) -> Identity {
Identity::new(access_key).expect("valid access key")
}
fn session(access_key: &str) -> Identity {
let binding = SessionBinding::new(crate::STS_SESSION_ISSUER, EXPIRES_AT).expect("valid binding");
long_term(access_key).with_session_binding(binding)
}
async fn principal_of(source: &IamPolicySource, identity: &Identity) -> PolicySnapshot {
source.snapshot(Some(identity)).await.expect("a snapshot")
}
fn payload(snapshot: &PolicySnapshot) -> &RustfsPrincipal {
snapshot.get::<RustfsPrincipal>().expect("the payload is a RustfsPrincipal")
}
fn refusal(result: Result<PolicySnapshot, PolicyError>) -> PolicyError {
match result {
Ok(snapshot) => panic!("expected a refusal, got a snapshot (empty: {})", snapshot.is_empty()),
Err(error) => error,
}
}
/// Compile-time: `RustfsPrincipal` has no `Debug`, so no formatting of a snapshot payload can
/// reach its claims. The two blanket impls make the path ambiguous, and so a compile error, as
/// soon as `RustfsPrincipal: Debug` holds.
#[test]
fn policy_principal_has_no_debug() {
trait AmbiguousIfDebug<A> {
fn some_item() {}
}
impl<T: ?Sized> AmbiguousIfDebug<()> for T {}
impl<T: ?Sized + core::fmt::Debug> AmbiguousIfDebug<u8> for T {}
let _ = <RustfsPrincipal as AmbiguousIfDebug<_>>::some_item;
}
/// Positive — a long-term user's facts: itself as username, both group sources, no claims,
/// not the owner.
#[tokio::test]
async fn policy_user_snapshot_carries_iam_facts() {
let iam = fake(true, || found(user()));
let snapshot = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
let principal = payload(&snapshot);
assert!(principal.kind() == PrincipalKind::User);
assert_eq!(principal.account(), ACCESS_KEY);
assert_eq!(principal.username(), ACCESS_KEY);
assert_eq!(principal.parent_user(), "");
assert_eq!(principal.credential_groups(), &Some(vec!["cred-group".to_owned()]));
assert_eq!(principal.group_memberships(), ["key-group".to_owned()]);
assert!(principal.claims().is_none());
assert!(!principal.is_owner());
}
/// Positive — an STS credential acts for its parent user, and its claims are the ones its
/// token carries, read with the root secret that signed it.
#[tokio::test]
async fn policy_sts_snapshot_carries_parent_user_and_claims() {
let token = sts_token(&[("sub", Value::from("alice-session"))], SIGNING_KEY);
let iam = fake(true, move || found(sts(PARENT, token.clone())));
let snapshot = principal_of(&source(iam), &session(ACCESS_KEY)).await;
let principal = payload(&snapshot);
assert!(principal.kind() == PrincipalKind::Temporary);
assert_eq!(principal.account(), ACCESS_KEY);
assert_eq!(principal.parent_user(), PARENT);
assert_eq!(principal.username(), PARENT);
let claims = principal.claims().expect("an STS principal has claims");
assert_eq!(claims.get("sub"), Some(&Value::from("alice-session")));
assert_eq!(principal.group_memberships(), ["parent-group".to_owned()]);
assert!(!principal.is_owner());
}
/// Positive — a service account's claims are read with its own secret, and it acts for its
/// parent.
#[tokio::test]
async fn policy_service_account_snapshot_reads_its_own_token() {
let mut token_claims: HashMap<String, Value> = HashMap::new();
token_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"));
token_claims.insert("parent".to_owned(), Value::from(PARENT));
let token = generate_jwt(&token_claims, SECRET).expect("a token");
let iam = fake(true, move || {
let mut cred = user();
cred.session_token = token.clone();
cred.parent_user = PARENT.to_owned();
cred.claims = Some([(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"))].into());
found(cred)
});
let snapshot = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
let principal = payload(&snapshot);
assert!(principal.kind() == PrincipalKind::ServiceAccount);
assert_eq!(principal.username(), PARENT);
assert_eq!(principal.claims().and_then(|c| c.get("parent")), Some(&Value::from(PARENT)));
assert_eq!(principal.group_memberships(), ["parent-group".to_owned()]);
assert!(!principal.is_owner());
}
/// Positive and negative — root is the owner without asking IAM, even before IAM is ready;
/// an STS credential of root is the owner too, but not once a session policy narrows it.
#[tokio::test]
async fn policy_owner_is_root_or_unrestricted_root_session() {
let not_ready = fake(false, || found(user()));
let snapshot = principal_of(&source(not_ready.clone()), &long_term(ROOT_ACCESS_KEY)).await;
let root = payload(&snapshot);
assert!(root.kind() == PrincipalKind::Root);
assert!(root.is_owner());
assert_eq!(not_ready.calls.load(Ordering::SeqCst), 0);
let plain = sts_token(&[], SIGNING_KEY);
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, plain.clone())));
assert!(payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
let narrowed = sts_token(&[(SESSION_POLICY_NAME, Value::from("e30"))], SIGNING_KEY);
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, narrowed.clone())));
assert!(!payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
}
/// Negative — an anonymous caller is an empty snapshot, and IAM is not asked.
#[tokio::test]
async fn policy_anonymous_is_empty_snapshot() {
let iam = fake(true, || found(user()));
let snapshot = source(iam.clone()).snapshot(None).await.expect("an anonymous snapshot");
assert!(snapshot.is_empty());
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
}
/// Negative — IAM that is not ready, not initialized, or failing is `unavailable`: never an
/// empty snapshot, which an authorizer would judge as an anonymous caller.
#[tokio::test]
async fn policy_iam_unavailable_is_unavailable() {
let not_ready = fake(false, || found(user()));
let error = refusal(source(not_ready.clone()).snapshot(Some(&long_term(ACCESS_KEY))).await);
assert_eq!(error, PolicyError::unavailable());
assert_eq!(not_ready.calls.load(Ordering::SeqCst), 0);
let failures: [fn() -> IamError; 2] = [
|| IamError::IamSysNotInitialized,
|| IamError::StringError("storage read failed".to_owned()),
];
for failure in failures {
let iam = fake(true, move || Err(failure()));
let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
assert_eq!(error, PolicyError::unavailable(), "{:?}", failure());
}
}
/// Negative — a principal IAM no longer knows is refused as unknown, not as unavailable and
/// not as an empty snapshot.
#[tokio::test]
async fn policy_unknown_principal_is_refused() {
let answers: [fn() -> KeyAnswer; 2] = [|| Ok((None, false)), || Err(IamError::NoSuchUser(ACCESS_KEY.to_owned()))];
for answer in answers {
let error = refusal(source(fake(true, answer)).snapshot(Some(&long_term(ACCESS_KEY))).await);
assert_eq!(error, PolicyError::because(UNKNOWN_PRINCIPAL));
}
}
/// Negative — a disabled or invalid principal is refused rather than given a snapshot.
#[tokio::test]
async fn policy_disabled_principal_is_refused() {
let invalid = fake(true, || Ok((Some(UserIdentity::new(user())), false)));
let error = refusal(source(invalid).snapshot(Some(&long_term(ACCESS_KEY))).await);
assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL));
let off = fake(true, || {
let mut cred = user();
cred.status = "off".to_owned();
found(cred)
});
let error = refusal(source(off).snapshot(Some(&long_term(ACCESS_KEY))).await);
assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL));
}
/// Negative — an STS token not signed by root yields no claims at all: it is `malformed`.
#[tokio::test]
async fn policy_sts_token_with_foreign_signature_is_malformed() {
let forged = sts_token(&[("sub", Value::from("alice-session"))], "not-the-root-secret");
let iam = fake(true, move || found(sts(PARENT, forged.clone())));
let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
assert_eq!(error, PolicyError::malformed());
}
/// Negative — the identity's credential kind must match what IAM holds: a session identity for
/// a long-term key, or a long-term identity for an STS key, is refused.
#[tokio::test]
async fn policy_session_kind_mismatch_is_refused() {
let token = sts_token(&[], SIGNING_KEY);
let iam = fake(true, move || found(sts(PARENT, token.clone())));
let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
let iam = fake(true, || found(user()));
let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
let iam = fake(false, || found(user()));
let error = refusal(source(iam).snapshot(Some(&session(ROOT_ACCESS_KEY))).await);
assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
}
fn service_account(parent: &str, extra_claims: &[(&str, Value)]) -> KeyAnswer {
let mut token_claims: HashMap<String, Value> = extra_claims.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect();
token_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"));
let mut cred = user();
cred.session_token = generate_jwt(&token_claims, SECRET).expect("a token");
cred.parent_user = parent.to_owned();
cred.claims = Some([(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"))].into());
found(cred)
}
/// Negative — a valid-flagged identity whose secret is empty or short gets no snapshot.
#[tokio::test]
async fn policy_short_secret_principal_is_refused() {
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
let iam = fake(true, move || {
let mut cred = user();
cred.secret_key = weak.to_owned();
found(cred)
});
let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL), "{weak:?}");
}
for usable in ["12345678", "abc\0defg"] {
let iam = fake(true, move || {
let mut cred = user();
cred.secret_key = usable.to_owned();
found(cred)
});
assert!(payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).kind() == PrincipalKind::User);
}
}
/// Negative — a root session issued through external OIDC is not the owner, though its parent
/// is root; the same session without the OIDC issuer is.
#[tokio::test]
async fn policy_oidc_root_session_is_not_owner() {
let oidc = sts_token(
&[
("iss", Value::from("rustfs-oidc")),
("oidc_provider", Value::from("corp-idp")),
("sub", Value::from("alice@corp")),
],
SIGNING_KEY,
);
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, oidc.clone())));
assert!(!payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
let local = sts_token(
&[
("iss", Value::from("rustfs-sts")),
("oidc_provider", Value::from("corp-idp")),
("sub", Value::from("alice@corp")),
],
SIGNING_KEY,
);
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, local.clone())));
assert!(payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
}
/// Positive and negative — a service account of root is the owner; one narrowed by a session
/// policy is not, and neither is a service account of an ordinary user.
#[tokio::test]
async fn policy_service_account_of_root_is_owner() {
let iam = fake(true, || service_account(ROOT_ACCESS_KEY, &[]));
let principal = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
assert!(payload(&principal).kind() == PrincipalKind::ServiceAccount);
assert!(payload(&principal).is_owner());
let iam = fake(true, || service_account(ROOT_ACCESS_KEY, &[(SESSION_POLICY_NAME, Value::from("e30"))]));
assert!(!payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).is_owner());
let iam = fake(true, || service_account(PARENT, &[]));
assert!(!payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).is_owner());
}
/// Negative — STS claims are read with the STS signing key, not with the root secret: a token
/// signed with the root secret while the signing key differs is `malformed`.
#[tokio::test]
async fn policy_sts_claims_use_the_signing_key() {
let root_signed = sts_token(&[("sub", Value::from("alice-session"))], ROOT_SECRET);
let iam = fake(true, move || found(sts(PARENT, root_signed.clone())));
let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
assert_eq!(error, PolicyError::malformed());
}
/// Negative — without root credentials no principal can be judged: `unavailable`, never an
/// empty snapshot, and IAM is not asked.
#[tokio::test]
async fn policy_without_root_is_unavailable() {
let iam = fake(true, || found(user()));
let source = IamPolicySource::new(iam.clone(), Arc::new(RootCell::new(None)));
for identity in [long_term(ACCESS_KEY), long_term(ROOT_ACCESS_KEY)] {
assert_eq!(refusal(source.snapshot(Some(&identity)).await), PolicyError::unavailable());
}
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
}
type LiveRoot = Arc<std::sync::Mutex<Option<(String, String)>>>;
/// The primary path: a source that reads a live accessor on every call, as the binary's does,
/// with nothing cached in between.
fn live_root_source(live: &LiveRoot) -> Arc<dyn RootSource> {
let accessor = live.clone();
Arc::new(move || {
let guard = accessor.lock().ok()?;
let (access_key, secret_key) = guard.as_ref()?;
RootCredentials::new(access_key, secret_key, SIGNING_KEY).ok().map(Arc::new)
})
}
fn set_live(live: &LiveRoot, value: Option<(&str, &str)>) {
*live.lock().expect("live root lock") = value.map(|(ak, sk)| (ak.to_owned(), sk.to_owned()));
}
/// Positive and negative — the two adapters read one live root accessor: a root change is seen
/// by both on the next request with no second step, the old root key stops being root to
/// both, and an uninitialized or unusable root makes both unavailable.
#[tokio::test]
async fn adapters_share_one_live_root() {
use rustfs_gateway::{CredentialLookup, CredentialProvider, ProviderError};
let live: LiveRoot = Arc::default();
set_live(&live, Some((ROOT_ACCESS_KEY, ROOT_SECRET)));
let iam = fake(true, || Ok((None, false)));
let adapters = crate::IamAdapters::new(iam, live_root_source(&live));
let found = adapters.credentials.lookup(ROOT_ACCESS_KEY).await;
assert!(matches!(found, Ok(CredentialLookup::Found(_))));
let snapshot = principal_of(&adapters.policy, &long_term(ROOT_ACCESS_KEY)).await;
assert!(payload(&snapshot).is_owner());
set_live(&live, Some(("newroot", "new-root-secret-01")));
match adapters.credentials.lookup("newroot").await {
Ok(CredentialLookup::Found(credentials)) => assert_eq!(credentials.secret().expose(), b"new-root-secret-01"),
_ => panic!("the new root key must resolve to the new secret"),
}
assert!(matches!(
adapters.credentials.lookup(ROOT_ACCESS_KEY).await,
Ok(CredentialLookup::NotFound)
));
assert!(payload(&principal_of(&adapters.policy, &long_term("newroot")).await).is_owner());
let old = refusal(adapters.policy.snapshot(Some(&long_term(ROOT_ACCESS_KEY))).await);
assert_eq!(old, PolicyError::because(UNKNOWN_PRINCIPAL));
for unusable in [None, Some(("newroot", "\0\0\0\0\0\0\0\0"))] {
set_live(&live, unusable);
assert!(matches!(adapters.credentials.lookup("newroot").await, Err(ProviderError::Unavailable)));
assert_eq!(
refusal(adapters.policy.snapshot(Some(&long_term("newroot"))).await),
PolicyError::unavailable()
);
}
}
/// Positive — `install` builds the authenticator exactly once, over a provider that reads the
/// pair's root source.
#[tokio::test]
async fn adapters_install_hands_over_the_paired_provider_once() {
use rustfs_gateway::{CredentialLookup, CredentialProvider, RegionSet, ServiceBuilder, SigV4Authenticator};
let live: LiveRoot = Arc::default();
set_live(&live, Some((ROOT_ACCESS_KEY, ROOT_SECRET)));
let adapters = crate::IamAdapters::new(fake(true, || Ok((None, false))), live_root_source(&live));
let mut handed = Vec::new();
let _builder = adapters.install(ServiceBuilder::new(), |provider| {
handed.push(provider.clone());
SigV4Authenticator::new(provider, RegionSet::new(["us-east-1"]).expect("a region"))
});
assert_eq!(handed.len(), 1);
let found = handed[0].lookup(ROOT_ACCESS_KEY).await;
assert!(matches!(found, Ok(CredentialLookup::Found(_))));
}
}
-216
View File
@@ -1,216 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The deployment's root credentials, as every adapter in this crate reads them.
//!
//! Responsible for: [`RootCredentials`], the validated root key pair plus the key STS session
//! tokens are signed with; [`RootSource`], the one live handle both the credential provider and
//! the policy source read on every request; and the effective-secret-length rule every secret in
//! this crate is held to.
//! NOT responsible for: where the root keys come from or when they change. The binary implements
//! [`RootSource`] directly over its live root accessor — a closure is enough — so there is no
//! second copy of root to keep in step.
//! Upstream: the binary's runtime credentials. Downstream: `crate::credentials`, `crate::policy`.
//!
//! # Why one source
//!
//! Legacy reads the current root credentials on every request. Two copies taken at assembly could
//! disagree, or go stale after a root change, and a key one adapter still treats as root would be
//! answered with the root secret or judged as the owner. Both adapters therefore hold the same
//! `Arc<dyn RootSource>`, handed to them together by [`crate::IamAdapters::new`].
use std::sync::Arc;
use rustfs_gateway::Credentials;
/// The shortest effective secret key, in bytes, that any credential in this crate may verify a
/// signature with. The same bound IAM applies to its own credentials; a signature over a shorter
/// (or empty) key is one anybody can compute.
pub const MIN_SECRET_KEY_LEN: usize = 8;
/// The number of bytes of `secret` a signature actually depends on: its length after trailing NUL
/// bytes are removed.
///
/// HMAC pads a key shorter than its block with zero bytes, and SigV4's first key is `"AWS4"`
/// followed by the secret, so trailing NULs add nothing: eight NUL bytes sign exactly like the
/// empty secret, and `"abc"` followed by five NULs signs like `"abc"`. A secret of NULs only has
/// an effective length of zero. An interior NUL is followed by real key bytes, which change the
/// padded key, so it counts like any other byte.
pub(crate) fn effective_secret_len(secret: &[u8]) -> usize {
secret.iter().rposition(|byte| *byte != 0).map_or(0, |last| last + 1)
}
/// Whether `secret` may verify a signature: its [`effective_secret_len`] is at least
/// [`MIN_SECRET_KEY_LEN`]. The one rule for root, IAM credentials and the STS signing key.
pub(crate) fn is_usable_secret(secret: &[u8]) -> bool {
effective_secret_len(secret) >= MIN_SECRET_KEY_LEN
}
/// Why [`RootCredentials::new`] refused its input. Carries no input text.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RootCredentialsError {
/// The access key id is not one the gateway can hold.
InvalidAccessKey,
/// The root secret key fails the effective-length rule.
WeakSecretKey,
/// The STS token-signing key fails the effective-length rule.
WeakSigningKey,
}
impl core::fmt::Display for RootCredentialsError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.write_str(match self {
Self::InvalidAccessKey => "root access key is not usable",
Self::WeakSecretKey => "root secret key is shorter than the minimum length",
Self::WeakSigningKey => "STS token-signing key is shorter than the minimum length",
})
}
}
impl std::error::Error for RootCredentialsError {}
/// The root key pair and the STS token-signing key, validated together.
///
/// Implements no `Debug`; the secrets are readable only inside this crate.
pub struct RootCredentials {
credentials: Credentials,
sts_signing_key: Box<str>,
}
impl RootCredentials {
/// Validates the root credentials and the key IAM signs STS session tokens with (today the
/// root secret, `rustfs_iam::manager::get_token_signing_key`; pass it separately so a future
/// key separation changes no adapter).
///
/// # Errors
///
/// [`RootCredentialsError`] when either secret fails the effective-length rule (fewer than
/// [`MIN_SECRET_KEY_LEN`] bytes once trailing NUL bytes are removed) or the access key id is not
/// one the gateway can hold.
pub fn new(access_key: &str, secret_key: &str, sts_signing_key: &str) -> Result<Self, RootCredentialsError> {
if !is_usable_secret(secret_key.as_bytes()) {
return Err(RootCredentialsError::WeakSecretKey);
}
if !is_usable_secret(sts_signing_key.as_bytes()) {
return Err(RootCredentialsError::WeakSigningKey);
}
let credentials =
Credentials::new(access_key, secret_key.as_bytes()).map_err(|_| RootCredentialsError::InvalidAccessKey)?;
Ok(Self {
credentials,
sts_signing_key: sts_signing_key.into(),
})
}
/// The root access key id, a public identifier.
#[must_use]
pub fn access_key(&self) -> &str {
self.credentials.identity().access_key_id()
}
pub(crate) fn credentials(&self) -> &Credentials {
&self.credentials
}
pub(crate) fn sts_signing_key(&self) -> &str {
&self.sts_signing_key
}
}
/// The live root credentials, read on every request by both adapters.
///
/// The binary implements it directly over RustFS's live root accessor, so a root change needs no
/// second step. Any `Fn() -> Option<Arc<RootCredentials>>` is a source; for example, over the
/// server context:
///
/// ```ignore
/// let root = move || {
/// let cred = ctx.action_credentials().get()?;
/// let signing_key = ctx.token_signing_key()?;
/// RootCredentials::new(&cred.access_key, &cred.secret_key, &signing_key).ok().map(Arc::new)
/// };
/// let adapters = IamAdapters::new(iam, Arc::new(root));
/// ```
///
/// Root credentials that fail validation read as `None`, which both adapters answer as
/// unavailable.
pub trait RootSource: Send + Sync + 'static {
/// The current root credentials; `None` while they are not initialized, which both adapters
/// answer as unavailable rather than guessing.
fn current(&self) -> Option<Arc<RootCredentials>>;
}
impl<F> RootSource for F
where
F: Fn() -> Option<Arc<RootCredentials>> + Send + Sync + 'static,
{
fn current(&self) -> Option<Arc<RootCredentials>> {
self()
}
}
/// Test helper: a [`RootSource`] holding fixed root credentials. Production code reads the live
/// accessor instead, so that there is no copy of root to keep in step.
#[cfg(test)]
pub(crate) struct RootCell(Option<Arc<RootCredentials>>);
#[cfg(test)]
impl RootCell {
pub(crate) fn new(root: Option<RootCredentials>) -> Self {
Self(root.map(Arc::new))
}
}
#[cfg(test)]
impl RootSource for RootCell {
fn current(&self) -> Option<Arc<RootCredentials>> {
self.0.clone()
}
}
#[cfg(test)]
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
mod tests {
use super::*;
/// Negative — an empty or short root secret is refused at construction, so no source can ever
/// hand out a root key anybody could sign with; eight bytes is accepted.
#[test]
fn root_short_secret_is_refused() {
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0", "abcdefg\0"] {
let refused = RootCredentials::new("rootadmin", weak, "signing-key-0001");
assert!(matches!(refused, Err(RootCredentialsError::WeakSecretKey)), "{} bytes", weak.len());
}
assert!(RootCredentials::new("rootadmin", "12345678", "signing-key-0001").is_ok());
}
/// Positive — an interior NUL is followed by real key bytes and counts: `"abc\0defg"` has an
/// effective length of eight and is accepted, as is the same secret with trailing NULs.
#[test]
fn root_interior_nul_counts_toward_length() {
for secret in ["abc\0defg", "abc\0defg\0\0"] {
assert!(RootCredentials::new("rootadmin", secret, "signing-key-0001").is_ok(), "{secret:?}");
assert!(RootCredentials::new("rootadmin", "root-secret-key-0001", secret).is_ok(), "{secret:?}");
}
}
/// Negative — an empty or short STS signing key is refused at construction.
#[test]
fn root_short_signing_key_is_refused() {
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
let refused = RootCredentials::new("rootadmin", "root-secret-key-0001", weak);
assert!(matches!(refused, Err(RootCredentialsError::WeakSigningKey)), "{} bytes", weak.len());
}
}
}
+20 -33
View File
@@ -425,10 +425,7 @@ enum FolderScanSource {
#[derive(Clone, Copy, PartialEq, Eq)]
enum FolderScanMode {
Namespace,
ErasureCandidate {
reuse_namespace: bool,
parent_has_metadata: bool,
},
ErasureCandidate { reuse_namespace: bool },
}
#[derive(Debug, PartialEq, Eq)]
@@ -2068,9 +2065,7 @@ impl FolderScanner {
if candidate_unclassified {
// Apply namespace reuse only after the current listing
// proves there is no direct erasure part.
if let FolderScanMode::ErasureCandidate {
reuse_namespace: true, ..
} = mode
if let FolderScanMode::ErasureCandidate { reuse_namespace: true } = mode
&& !into.compacted
&& self.old_cache.find(&folder.name).is_some()
{
@@ -2084,23 +2079,10 @@ impl FolderScanner {
return Ok(FolderScanOutcome::Namespace);
}
candidate_unclassified = false;
// An empty UUID directory beside object metadata has
// no namespace work. Its listing was already charged;
// nonempty namespaces still need directory admission.
let empty_object_child = matches!(
mode,
FolderScanMode::ErasureCandidate {
parent_has_metadata: true,
..
}
) && deferred_metadata.is_none()
&& queued_folders.is_empty();
if !empty_object_child {
if !self.budget.try_start_directory() {
return Err(ScannerError::Other("Operation cancelled".to_string()));
}
wait_for_scanner_io(&ctx, self.sleeper.sleep_folder_preset()).await?;
if !self.budget.try_start_directory() {
return Err(ScannerError::Other("Operation cancelled".to_string()));
}
wait_for_scanner_io(&ctx, self.sleeper.sleep_folder_preset()).await?;
}
let Some(entry) = deferred_metadata.take() else { break };
(entry, true)
@@ -2570,6 +2552,20 @@ impl FolderScanner {
return Err(ScannerError::Other("Operation cancelled".to_string()));
}
if found_object_metadata && self.is_erasure_mode {
// If we found an object in erasure mode, we skip subdirs (only datadirs)...
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
folder = %folder.name,
state = "erasure_object_found",
"Scanner folder descent stopped after erasure object"
);
break;
}
// Checkpointed sweeps need child entries to validate their resume
// frontier (#8292). Outside that, also compact a wide bucket root:
// excluding it left #8961's ~170k top-level prefixes unbounded until
@@ -2658,7 +2654,6 @@ impl FolderScanner {
let mut child_mode = if self.is_erasure_mode && is_erasure_candidate(&folder_item.name) {
FolderScanMode::ErasureCandidate {
reuse_namespace: reuse_covered,
parent_has_metadata: found_object_metadata,
}
} else {
FolderScanMode::Namespace
@@ -2726,10 +2721,7 @@ impl FolderScanner {
}
// Sampling may skip namespace work, never the
// classification that protects its parent object.
child_mode = FolderScanMode::ErasureCandidate {
reuse_namespace: true,
parent_has_metadata: found_object_metadata,
};
child_mode = FolderScanMode::ErasureCandidate { reuse_namespace: true };
} else {
folder_item.object_heal_prob_div = data_usage_update_dir_cycles();
}
@@ -2824,11 +2816,6 @@ impl FolderScanner {
updates.cache.delete_recursive(&h);
}
into.children.remove(&clone_cache_value(&h.0));
// Object keys can also be prefixes. A classified payload
// directory must not hide their other namespace children.
if found_object_metadata {
continue;
}
// Earlier namespace siblings may have completed, but none
// can certify coverage of this unresolved parent object.
self.coverage_gap |= forward_sweep;
@@ -183,117 +183,6 @@ fn charged_ops(root: &std::path::Path) -> u64 {
status.charged_ops
}
#[tokio::test]
#[serial]
async fn erasure_object_metadata_preserves_child_object_namespaces() {
for (corrupt_parent, cached_parent) in [(false, false), (true, false), (false, true)] {
let (mut scanner, root) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, root.clone());
scanner.old_cache.info.name = "bucket".into();
scanner.new_cache.info.name = "bucket".into();
scanner.is_erasure_mode = true;
bind_budget(&mut scanner, &root, 1_000_000);
let namespace = format!("parent/{}/nested", Uuid::new_v4());
for object in ["parent", "parent/child", namespace.as_str()] {
write_test_object_metadata_bytes(&root, "bucket", object, &metadata_for_object_version("bucket", object, None)).await;
}
let payload = root.join("bucket/parent").join(Uuid::new_v4().to_string());
tokio::fs::create_dir_all(&payload)
.await
.expect("create parent payload directory");
tokio::fs::write(payload.join("part.1"), b"shard")
.await
.expect("create actual payload evidence");
if corrupt_parent {
tokio::fs::write(root.join("bucket/parent/xl.meta"), b"")
.await
.expect("corrupt only the parent metadata");
}
if cached_parent {
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
scanner.old_cache.replace(
"bucket/parent",
"bucket",
DataUsageEntry {
objects: 1,
size: 1,
compacted: true,
..Default::default()
},
);
let interval = data_usage_update_dir_cycles();
scanner.old_cache.info.next_cycle = u64::from(
(0..interval)
.find(|cycle| hash_path("bucket/parent").mod_(*cycle, interval))
.expect("one normal sampling cycle revisits the old parent-only cache"),
);
}
let mut into = DataUsageEntry::default();
scanner
.scan_folder(
CancellationToken::new(),
CachedFolder {
name: "bucket".into(),
parent: None,
object_heal_prob_div: 1,
},
&mut into,
)
.await
.expect("scan object and child namespaces independently");
let total = scanner.new_cache.size_recursive("bucket").expect("retain bucket accounting");
assert_eq!(total.objects, if corrupt_parent { 2 } else { 3 });
assert_eq!(total.size, if corrupt_parent { 2 } else { 3 });
assert_eq!(total.failed_objects, usize::from(corrupt_parent));
assert_eq!(
scanner.new_cache.info.failed_objects.len(),
usize::from(corrupt_parent),
"payload evidence must not invent missing parent metadata"
);
}
}
#[tokio::test]
#[serial]
async fn erasure_object_prefix_children_keep_the_object_budget() {
let (mut scanner, root) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, root.clone());
scanner.old_cache.info.name = "bucket".into();
scanner.new_cache.info.name = "bucket".into();
scanner.is_erasure_mode = true;
bind_budget(&mut scanner, &root, 1_000_000);
for object in ["parent", "parent/child", "parent/other"] {
write_test_object_metadata(&root, "bucket", object).await;
}
let visits = metadata_visits::observe(&mut scanner);
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
crate::scanner_budget::ScannerCycleBudgetConfig {
max_objects: Some(2),
..Default::default()
},
);
scanner.budget = budget.clone();
let result = scanner
.scan_folder(
budget.token(),
CachedFolder {
name: "bucket".into(),
parent: None,
object_heal_prob_div: 1,
},
&mut DataUsageEntry::default(),
)
.await;
assert!(result.is_err(), "parent metadata cannot hide children or bypass their budget");
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects));
assert_eq!(budget.progress().0, 2);
assert_eq!(visits.load(Ordering::Relaxed), 2);
}
#[tokio::test]
#[serial]
async fn erasure_directory_probe_reaches_eof_across_budgeted_batches() {
@@ -315,10 +204,7 @@ async fn erasure_directory_probe_reaches_eof_across_budgeted_batches() {
ctx,
folder,
&mut DataUsageEntry::default(),
FolderScanMode::ErasureCandidate {
reuse_namespace: false,
parent_has_metadata: false,
},
FolderScanMode::ErasureCandidate { reuse_namespace: false },
false,
)
.await
@@ -385,10 +271,7 @@ async fn scanner_directory_probe_budget_wait_cancels_without_rate_delay() {
ctx.clone(),
folder,
&mut DataUsageEntry::default(),
FolderScanMode::ErasureCandidate {
reuse_namespace: false,
parent_has_metadata: false,
},
FolderScanMode::ErasureCandidate { reuse_namespace: false },
false,
)
.await;
@@ -428,10 +311,7 @@ async fn scanner_directory_probe_cancels_during_entry_budget_wait() {
ctx.clone(),
folder,
&mut DataUsageEntry::default(),
FolderScanMode::ErasureCandidate {
reuse_namespace: false,
parent_has_metadata: false,
},
FolderScanMode::ErasureCandidate { reuse_namespace: false },
false,
)
.await;
@@ -502,16 +382,7 @@ async fn erasure_candidate_defers_metadata_until_part_classification_finishes()
scanner.budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, Default::default());
let mut into = DataUsageEntry::default();
let outcome = scanner
.scan_folder_inner(
ctx,
folder,
&mut into,
FolderScanMode::ErasureCandidate {
reuse_namespace: false,
parent_has_metadata: false,
},
false,
)
.scan_folder_inner(ctx, folder, &mut into, FolderScanMode::ErasureCandidate { reuse_namespace: false }, false)
.await
.expect("classify candidate");
assert_eq!(outcome, FolderScanOutcome::ErasureData);
@@ -723,10 +594,7 @@ async fn erasure_candidate_part_symlink_is_not_data_evidence() {
CancellationToken::new(),
folder,
&mut DataUsageEntry::default(),
FolderScanMode::ErasureCandidate {
reuse_namespace: false,
parent_has_metadata: false,
},
FolderScanMode::ErasureCandidate { reuse_namespace: false },
false,
)
.await
-1
View File
@@ -54,7 +54,6 @@ The four site-replication state-writer concurrency proofs reserve the available
| PR, non-doc change | `Typos` | `ci.yml` `typos` | Via aggregate | `typos` |
| Full selection | `Workspace tests` (serial ILM step) | `ci.yml` `workspace-tests` | Via aggregate | exact `Run ignored ILM integration tests serially` command in the job |
| Nightly schedule, manual dispatch | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)`, `Test and Lint (http3)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo nextest run` with the job's `--features` |
| Full selection, every event | `Test and Lint (gateway-stack)` | `ci.yml` `test-and-lint-gateway-stack` | Via aggregate | `cargo clippy -p rustfs --all-targets --features gateway-stack -- -D warnings`; `cargo nextest run -p rustfs --features gateway-stack` with the job's filter |
| Nightly schedule, manual dispatch, full selection | `Connect Feature Boundaries` | `ci.yml` `connect-feature-boundaries` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail; `scripts/check_offline_enrollment_e2e.sh` |
| PR, non-doc change | `Build RustFS Debug Binary` | `ci.yml` `build-rustfs-debug-binary` | Via aggregate; prerequisite for black-box jobs | `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks` (binary plus its `rustfs.e2e.json` sidecar) |
| Nightly schedule, manual dispatch, full selection | `io_uring Integration (real)` | `ci.yml` `uring-integration` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
+1 -16
View File
@@ -56,12 +56,6 @@ required-features = ["swift"]
name = "swift_object_integration_test"
required-features = ["swift"]
# Starts an embedded server on the gateway stack, which only a build with the
# feature can select.
[[test]]
name = "embedded_gateway_stack_test"
required-features = ["gateway-stack"]
[features]
default = ["ftps", "webdav", "gcs"]
fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips", "rustfs-ecstore/fips", "rustfs-protocols/fips", "rustfs-s3-client/fips", "rustfs-targets/fips", "rustfs-utils/fips"]
@@ -75,19 +69,12 @@ license = []
io-scheduler-debug = [] # Enable debug information in I/O scheduler
tracing-chunk-debug = [] # Enable per-chunk tracing in data plane (high noise, for debugging only)
full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope", "gcs"]
# The debug-only E2E binary also carries the gateway S3 stack, so the gateway smoke
# lane runs the same artifact as the legacy one with RUSTFS_S3_STACK as its only change.
e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hooks", "gateway-stack"]
e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hooks"]
# Shortens Connect credentials only in debug E2E builds.
connect-e2e-short-credentials = []
# Builds the dedicated rustfs-cli-e2e target with a build-time public enrollment root.
offline-enrollment-e2e-root = []
rio-v2 = ["rustfs-ecstore/rio-v2"]
# The rustfs-gateway S3 stack, selected at startup with RUSTFS_S3_STACK=gateway
# (rustfs/backlog#2734). Under construction: off by default and never enabled in a
# release build; the stack refuses to compile without debug assertions, and
# `scripts/check_gateway_stack_release_boundary.sh` keeps it out of release inputs.
gateway-stack = ["dep:rustfs-gateway", "dep:rustfs-gateway-adapter"]
pyroscope = ["rustfs-obs/pyroscope", "dep:pyroscope", "dep:object", "dep:symbolic-demangle"]
# Tokio runtime telemetry. Requires `--cfg tokio_unstable`; use `make build-profiling`.
dial9 = ["rustfs-obs/dial9"]
@@ -268,8 +255,6 @@ rustfs-notify = { workspace = true }
rustfs-obs = { workspace = true }
rustfs-policy = { workspace = true }
rustfs-protocols = { workspace = true }
rustfs-gateway = { workspace = true, optional = true }
rustfs-gateway-adapter = { workspace = true, optional = true }
rustfs-gateway-types = { workspace = true, features = ["compat-s3s-rustfs"] }
rustfs-protos = { workspace = true }
rustfs-rio = { workspace = true }
+1 -2
View File
@@ -165,8 +165,7 @@ impl Operation for ChangeOwnPasswordHandler {
let signing_secret = req
.credentials()
.ok_or_else(|| s3::error(S3ErrorCode::AccessDenied, "authentication required"))?
.require_secret_key()
.map_err(|err| s3::error(S3ErrorCode::InternalError, err.to_string()))?
.secret_key()
.to_string();
let caller = CallerIdentity::resolve_envelope(req.envelope()).await?;
let audit = AccountAuditContext::from_envelope(req.envelope());
+2 -4
View File
@@ -63,10 +63,8 @@ impl Operation for RecoverOrphanedBucketHandler {
authorize_admin_envelope(req.envelope(), vec![Action::AdminAction(AdminAction::RecoverOrphanedBucketAction)]).await?;
let secret_key = req
.credentials()
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "authentication required"))?
.require_secret_key()
.map(|secret| Zeroizing::new(secret.to_owned()))
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, err.to_string()))?;
.map(|credentials| Zeroizing::new(credentials.secret_key().to_owned()))
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "authentication required"))?;
let bucket = params
.get("bucket")
.filter(|bucket| !bucket.is_empty())
+17 -122
View File
@@ -15,12 +15,11 @@
use crate::admin::operation::{AdminRequest, AdminResponse, Operation};
use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, S3Router};
use crate::admin::runtime_sources::{ServerContextSlot, current_action_credentials, current_object_store_handle, current_region};
use crate::admin::storage_api::access::{ReqInfo, authorize_owned_request, authorize_request};
use crate::admin::storage_api::access::{ReqInfo, authorize_request};
use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
use crate::admin::storage_api::contract::list::ListOperations as _;
use crate::admin::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _};
use crate::admin::storage_api::object::StorageObjectOptions as ObjectOptions;
use crate::app::{RequestEnvelope, RequestHead};
use crate::auth::{check_key_valid, get_session_token};
use crate::error::ApiError;
use crate::license::license_check;
@@ -496,32 +495,25 @@ async fn authorize_object_zip_download_s3_action(
req_info.version_id = None;
}
let mut extensions = http::Extensions::new();
extensions.insert(req.remote_addr().map(RemoteAddr));
if let Some(client_info) = req.client_info() {
extensions.insert(client_info.clone());
}
if let Some(server_context) = req.envelope().server_context() {
extensions.insert(server_context.clone());
}
if let Some(req_info) = req.req_info() {
extensions.insert(req_info.clone());
}
let mut authorization = RequestEnvelope::from_parts(
RequestHead {
method: req.method().clone(),
uri: req.uri().clone(),
headers: req.headers().clone(),
credentials: None,
region: None,
service: None,
},
extensions,
let mut authorization = s3_authorization_request(
req.method().clone(),
req.uri().clone(),
&authenticated_zip_download_auth_context(req),
req.envelope().server_context().cloned(),
req.req_info().cloned(),
);
authorize_owned_request(&mut authorization, Action::S3Action(action), None).await
authorize_request(&mut authorization, Action::S3Action(action))
.await
.map_err(Into::into)
}
/// Builds the legacy authorization request used when a download token is redeemed.
/// The S3 gate (`storage::access::authorize_request`) still evaluates the s3s
/// request shape, so the facts it reads are copied into one for the duration
/// of the call: the authenticated `ReqInfo`, the peer and proxy-reported client
/// addresses, the headers and query the policy conditions come from, and the
/// server slot that selects the store. The gate reads neither the presented
/// credentials nor the region or service, so those stay unset, and nothing it
/// attaches to the request is read afterwards.
fn s3_authorization_request(
method: Method,
uri: Uri,
@@ -951,103 +943,6 @@ mod tests {
use super::*;
use rustfs_s3_types::S3ErrorCode;
#[tokio::test]
async fn zip_scope_authorizes_real_iam_and_restores_prefix_query() {
use crate::admin::operation::test_support::TestRequest;
use crate::admin::runtime_sources::{AppContext, IamInterface, KmsInterface};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use rustfs_iam::store::{Store as _, UserType};
use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq};
use rustfs_policy::policy::Policy;
struct ReadyIam(Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>>);
impl IamInterface for ReadyIam {
fn handle(&self) -> Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>> {
Arc::clone(&self.0)
}
fn is_ready(&self) -> bool {
true
}
}
struct TestKms;
impl KmsInterface for TestKms {
fn handle(&self) -> Arc<rustfs_kms::KmsServiceManager> {
Arc::new(rustfs_kms::KmsServiceManager::new())
}
}
let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
store
.make_bucket("photos", &MakeBucketOptions::default())
.await
.expect("create bucket");
rustfs_iam::store::object::ObjectStore::new(Arc::clone(&store))
.save_iam_config(
serde_json::json!({"version": 1}),
format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX),
)
.await
.expect("seed IAM format");
let iam = rustfs_iam::build_iam_sys(Arc::clone(&store)).await.expect("initialize IAM");
let access_key = "zip-scope-user";
iam.create_user(
access_key,
&AddOrUpdateUserReq {
secret_key: "zip-scope-secret-key".to_owned(),
policy: None,
status: AccountStatus::Enabled,
},
)
.await
.expect("create user");
iam.set_policy(
"zip-scope-policy",
Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListBucket","Resource":"arn:aws:s3:::photos","Condition":{"StringEquals":{"s3:prefix":"allowed/"}}},{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/allowed/*"}]}"#)
.expect("parse policy"),
)
.await
.expect("store policy");
iam.policy_db_set(access_key, UserType::Reg, false, "zip-scope-policy")
.await
.expect("attach policy");
let credentials = iam.get_user(access_key).await.expect("load user").credentials;
let context = Arc::new(AppContext::new(store, Arc::new(ReadyIam(iam)), Arc::new(TestKms)));
let slot = ServerContextSlot::new();
assert!(slot.install(context));
let original_uri = "/rustfs/admin/v3/object-zip-downloads?prefix=old&marker=1";
let mut request = TestRequest::new(Method::POST, original_uri)
.extension(slot)
.extension(ReqInfo {
cred: Some(credentials),
..Default::default()
})
.build();
let mut selection = valid_request();
selection.prefix = None;
selection.prefixes = vec!["allowed/".to_owned()];
selection.objects = vec!["allowed/file".to_owned()];
authorize_object_zip_download_scope(&mut request, &selection)
.await
.expect("allowed prefix and object");
assert_eq!(request.uri().to_string(), original_uri);
selection.prefixes = vec!["denied/".to_owned()];
let err = authorize_object_zip_download_scope(&mut request, &selection)
.await
.expect_err("another prefix is denied");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
assert_eq!(request.uri().to_string(), original_uri);
selection.prefixes = vec!["allowed/".to_owned()];
selection.objects = vec!["denied/file".to_owned()];
let err = authorize_object_zip_download_scope(&mut request, &selection)
.await
.expect_err("another object is denied");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
assert_eq!(request.uri().to_string(), original_uri);
}
fn valid_request() -> CreateObjectZipDownloadRequest {
CreateObjectZipDownloadRequest {
bucket: "photos".to_string(),
+1 -1
View File
@@ -343,7 +343,7 @@ impl Operation for AddServiceAccount {
let (cred, owner) =
check_key_valid(get_session_token(req.uri(), req.headers()).unwrap_or_default(), req_cred.access_key()).await?;
if !req_cred.secret_key_matches(&cred.secret_key) {
if !constant_time_eq(&cred.secret_key, req_cred.secret_key()) {
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "access denied"));
}
+1 -1
View File
@@ -222,7 +222,7 @@ async fn handle_assume_role(
// The signature was verified before the request body was consumed. A fresh
// IAM lookup must not silently replace the secret that authenticated it.
if !user.secret_key_matches(&cred.secret_key) {
if !crate::auth::constant_time_eq(&cred.secret_key, user.secret_key()) {
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
}
+16 -11
View File
@@ -15,7 +15,7 @@
use crate::admin::operation::{AdminRequest, AdminResponse};
use crate::admin::runtime_sources;
use crate::admin::runtime_sources::default_admin_usecase;
use crate::admin::storage_api::access::{ReqInfo, authorize_owned_internal_object_request};
use crate::admin::storage_api::access::{ReqInfo, authorize_internal_object_request};
use crate::admin::storage_api::bucket::metadata::table_catalog_path_hash;
use crate::admin::storage_api::runtime::ECStore;
use crate::admin::utils::owned_empty_response;
@@ -1088,8 +1088,12 @@ fn build_sensitive_json_response<T: Serialize>(status: StatusCode, body: &T) ->
Ok(AdminResponse::with_headers(status, body, headers))
}
// These handlers still use the legacy router while their request bodies migrate.
// TODO(T1.12-codemod): move them to the owned operation boundary.
// The commit-path handlers (`TableCommitObjectBackend::for_request`) authorize
// exact object keys through `storage::access::authorize_internal_object_request`,
// which still takes the s3s request. Until that path has an owned form they keep
// the s3s request and read every other request fact through this owned view.
// TODO(T1.12-codemod): remove once the generic object authorization path moves
// to the request envelope.
type LegacyRequest = s3s::S3Request<s3s::Body>;
type LegacyResponse = s3s::S3Response<(StatusCode, s3s::Body)>;
@@ -1314,14 +1318,15 @@ fn install_table_catalog_s3_request_info(req: &mut LegacyRequest, principal: &Ta
}
async fn authorize_table_catalog_s3_actions(
req: &mut AdminRequest,
req: &mut LegacyRequest,
bucket: &str,
object: &str,
actions: &[S3Action],
) -> S3Result<()> {
let original = {
let req_info = req
.req_info_mut()
.extensions
.get_mut::<ReqInfo>()
.ok_or_else(|| s3_error!(AccessDenied, "authentication required"))?;
(
req_info.bucket.replace(bucket.to_string()),
@@ -1332,15 +1337,15 @@ async fn authorize_table_catalog_s3_actions(
let mut result = Ok(());
for action in actions {
if let Err(err) = authorize_owned_internal_object_request(req.envelope_mut(), Action::S3Action(*action)).await {
if let Err(err) = authorize_internal_object_request(req, Action::S3Action(*action)).await {
result = Err(err);
break;
}
}
if let Some(req_info) = req.req_info_mut() {
if let Some(req_info) = req.extensions.get_mut::<ReqInfo>() {
(req_info.bucket, req_info.object, req_info.version_id) = original;
}
result
result.map_err(Into::into)
}
async fn table_catalog_request_principal(req: &AdminRequest) -> S3Result<TableCatalogRequestPrincipal> {
@@ -1378,7 +1383,7 @@ async fn table_catalog_request_principal(req: &AdminRequest) -> S3Result<TableCa
#[derive(Clone)]
enum TableCommitObjectAuthorization {
Request(Arc<tokio::sync::Mutex<AdminRequest>>),
Request(Arc<tokio::sync::Mutex<LegacyRequest>>),
Preauthorized,
#[cfg(test)]
Test {
@@ -1454,7 +1459,7 @@ where
fn for_request(backend: B, req: LegacyRequest) -> Self {
Self::new(
backend,
TableCommitObjectAuthorization::Request(Arc::new(tokio::sync::Mutex::new(legacy_request_view(&req)))),
TableCommitObjectAuthorization::Request(Arc::new(tokio::sync::Mutex::new(req))),
rustfs_utils::get_env_bool(crate::table_catalog::ENV_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED, false),
)
}
@@ -2088,7 +2093,7 @@ where
let result = match &self.authorization {
TableCommitObjectAuthorization::Request(req) => {
let req = req.lock().await;
authorize_table_warehouse_location_request(&req, table_bucket, location).await
authorize_table_warehouse_location_request(&legacy_request_view(&req), table_bucket, location).await
}
TableCommitObjectAuthorization::Preauthorized => Ok(()),
#[cfg(test)]
@@ -77,7 +77,8 @@ fn table_catalog_handler_request(context: Arc<AppContext>, access_key: &str, sec
load_table_request_builder(context, access_key, secret_key).build()
}
/// Builds the legacy request accepted by the commit backend, which derives an owned view.
/// The s3s form of [`table_catalog_handler_request`] for the commit backend,
/// which still holds the legacy request (see `legacy_request_view`).
// TODO(T1.12-codemod): remove with `TableCommitObjectBackend::for_request`'s s3s parameter.
fn legacy_table_catalog_handler_request(
context: Arc<AppContext>,
@@ -1294,10 +1295,6 @@ async fn table_storage_credentials_are_rejected_by_catalog_authentication() {
#[tokio::test]
async fn warehouse_location_claim_checks_request_identity_and_relocation() {
let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
store
.make_bucket("warehouse", &MakeBucketOptions::default())
.await
.expect("create warehouse bucket");
rustfs_iam::store::object::ObjectStore::new(store.clone())
.save_iam_config(
serde_json::json!({"version": 1}),
@@ -1335,18 +1332,6 @@ async fn warehouse_location_claim_checks_request_identity_and_relocation() {
iam.policy_db_set(access_key, UserType::Reg, false, "location-creator")
.await
.expect("attach location policy");
iam.set_policy(
"owned-object-policy",
Policy::parse_config(
br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::warehouse/owned/*"},{"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::warehouse/owned/writable"}]}"#,
)
.expect("parse object policy"),
)
.await
.expect("store object policy");
iam.policy_db_set(access_key, UserType::Reg, false, "location-creator,owned-object-policy")
.await
.expect("attach object policy");
let context = Arc::new(AppContext::new(store, Arc::new(RequestIam { handle: iam }), Arc::new(RequestKms)));
assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
access_key: "location-root".to_string(),
@@ -1354,125 +1339,6 @@ async fn warehouse_location_claim_checks_request_identity_and_relocation() {
status: "on".to_string(),
..Default::default()
}));
// Exercise the actual owned internal authorization path, including its temporary target.
let mut legacy = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
let principal = table_catalog_request_principal(&legacy_request_view(&legacy))
.await
.expect("resolve principal");
install_table_catalog_s3_request_info(&mut legacy, &principal).expect("install authorization context");
let info = legacy.extensions.get_mut::<ReqInfo>().expect("request info");
info.bucket = Some("original-bucket".to_owned());
info.object = Some("original-object".to_owned());
info.version_id = Some("original-version".to_owned());
let mut owned = legacy_request_view(&legacy);
for (object, allowed) in [("owned/allowed", true), ("shared/denied", false)] {
let result = authorize_table_catalog_s3_actions(&mut owned, "warehouse", object, &[S3Action::GetObjectAction]).await;
if allowed {
result.expect("the exact owned object should be allowed");
} else {
assert_eq!(result.expect_err("the other object should be denied").code(), &S3ErrorCode::AccessDenied);
}
let info = owned.envelope().req_info().expect("request info remains installed");
assert_eq!(info.bucket.as_deref(), Some("original-bucket"));
assert_eq!(info.object.as_deref(), Some("original-object"));
assert_eq!(info.version_id.as_deref(), Some("original-version"));
}
// The exact-key check selects internal-object mode: a request header the object
// write does not apply (a malformed x-amz-tagging) must not reach the decision,
// while the ordinary owned entry point still evaluates it and rejects it.
let mut tagged = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
install_table_catalog_s3_request_info(&mut tagged, &principal).expect("install authorization context");
tagged
.headers
.insert("x-amz-tagging", http::HeaderValue::from_static("classification=%ZZ"));
let mut tagged = legacy_request_view(&tagged);
authorize_table_catalog_s3_actions(&mut tagged, "warehouse", "owned/writable", &[S3Action::PutObjectAction])
.await
.expect("internal-object mode ignores request options the write does not apply");
let info = tagged.req_info_mut().expect("request info");
info.bucket = Some("warehouse".to_owned());
info.object = Some("owned/writable".to_owned());
crate::app::authorize_owned_request(
tagged.envelope_mut(),
rustfs_policy::policy::action::Action::S3Action(S3Action::PutObjectAction),
None,
)
.await
.expect_err("ordinary mode evaluates the malformed tagging header");
let objects = TestTableCatalogObjectBackend::default();
objects
.seed_object("warehouse", "owned/read-only", b"readable".to_vec())
.await;
objects.seed_object("warehouse", "shared/denied", b"unchanged".to_vec()).await;
let backend = || {
let mut request = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
install_table_catalog_s3_request_info(&mut request, &principal).expect("install authorization context");
TableCommitObjectBackend::for_request(objects.clone(), request)
};
let allowed = backend();
let object = allowed
.read_object("warehouse", "owned/read-only")
.await
.expect("authorized read")
.expect("seeded object");
assert_eq!(object.data, b"readable");
allowed
.put_object(
"warehouse",
"owned/writable",
b"written".to_vec(),
crate::table_catalog::TableCatalogPutPrecondition::Any,
)
.await
.expect("authorized write");
assert_eq!(
objects
.read_object("warehouse", "owned/writable")
.await
.expect("read stored object")
.expect("written object")
.data,
b"written"
);
for (object, write) in [("shared/denied", false), ("owned/read-only", true), ("shared/denied", true)] {
let denied = backend();
let result = if write {
denied
.put_object(
"warehouse",
object,
b"modified".to_vec(),
crate::table_catalog::TableCatalogPutPrecondition::Any,
)
.await
.map(|_| ())
} else {
denied.read_object("warehouse", object).await.map(|_| ())
};
assert!(result.is_err(), "unauthorized {object} operation must fail");
assert_eq!(
denied.finish(Ok(())).await.expect_err("retain authorization error").code(),
&S3ErrorCode::AccessDenied
);
}
assert_eq!(
objects
.read_object("warehouse", "owned/read-only")
.await
.expect("read object")
.expect("seeded object")
.data,
b"readable"
);
assert_eq!(
objects
.read_object("warehouse", "shared/denied")
.await
.expect("read object")
.expect("seeded object")
.data,
b"unchanged"
);
let request = || table_catalog_handler_request(context.clone(), access_key, secret_key);
for location in ["s3://warehouse/owned", "s3://warehouse/owned/", "s3://warehouse/owned/child"] {
authorize_table_warehouse_location_request(&request(), "warehouse", location)
+1 -4
View File
@@ -243,10 +243,7 @@ impl Operation for AddUser {
let (cred, owner) =
check_key_valid(get_session_token(req.uri(), req.headers()).unwrap_or_default(), input_cred.access_key()).await?;
// The caller's current secret outlives the body read below.
let input_secret = input_cred
.require_secret_key()
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, err.to_string()))?
.to_owned();
let input_secret = input_cred.secret_key().to_owned();
let ak = query.access_key.as_deref().unwrap_or_default();
+11 -6
View File
@@ -210,10 +210,11 @@ pub(crate) mod test_support {
/// token is read from the request like the s3s edge does, so a test
/// that sets `x-amz-security-token` keeps working unchanged.
pub(crate) fn credentials(mut self, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Self {
self.credentials = Some(
RequestCredentials::new(access_key.to_owned(), secret_key.to_owned(), session_token.map(str::to_owned))
.expect("test credentials carry a non-empty secret"),
);
self.credentials = Some(RequestCredentials::new(
access_key.to_owned(),
secret_key.to_owned(),
session_token.map(str::to_owned),
));
self
}
@@ -245,7 +246,11 @@ pub(crate) mod test_support {
pub(crate) fn build(self) -> AdminRequest {
let credentials = self.credentials.map(|credentials| match credentials.session_token() {
Some(_) => credentials,
None => credentials.with_session_token(get_session_token(&self.uri, &self.headers).map(str::to_owned)),
None => RequestCredentials::new(
credentials.access_key().to_owned(),
credentials.secret_key().to_owned(),
get_session_token(&self.uri, &self.headers).map(str::to_owned),
),
});
let head = RequestHead {
method: self.method,
@@ -293,7 +298,7 @@ mod tests {
.build();
let credentials = req.credentials().expect("credentials were supplied");
assert_eq!(credentials.access_key(), "AKIATEST");
assert_eq!(credentials.secret_key(), Some("secret"));
assert_eq!(credentials.secret_key(), "secret");
assert_eq!(credentials.session_token(), Some("from-header"));
let explicit = TestRequest::new(Method::GET, "/rustfs/admin/v3/info")
+1 -1
View File
@@ -3882,7 +3882,7 @@ mod tests {
.credentials()
.ok_or_else(|| OwnedS3Error::with_message(OwnedS3ErrorCode::AccessDenied, "owned admin identity is required"))?;
let identity_matches = credentials.access_key() == "AKIAOWNED"
&& credentials.secret_key_matches("owned-secret")
&& constant_time_eq(credentials.secret_key(), "owned-secret")
&& credentials
.session_token()
.is_some_and(|token| constant_time_eq(token, "owned-session"));
+3 -2
View File
@@ -953,8 +953,9 @@ pub(crate) mod data_usage {
}
pub(crate) mod access {
pub(crate) use crate::app::{authorize_owned_internal_object_request, authorize_owned_request};
pub(crate) use crate::storage::storage_api::access_consumer::{ReqInfo, authorize_request};
pub(crate) use crate::storage::storage_api::access_consumer::{
ReqInfo, authorize_internal_object_request, authorize_request,
};
pub(crate) use crate::storage::storage_api::request_context_consumer::{RequestContext, spawn_traced};
}
+60 -162
View File
@@ -19,8 +19,8 @@ use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
#[cfg(test)]
use super::storage_api::bucket_usecase::access::ReqInfo;
use super::storage_api::bucket_usecase::access::{
ListBucketsAuthzRequest, TableDataPlaneListAccess, authorize_request, bucket_config_mutation_incarnation,
log_list_buckets_iam_implicit_deny, prepare_list_buckets_iam_authorization, prepare_odm_read_generation, req_info_ref,
TableDataPlaneListAccess, authorize_request, bucket_config_mutation_incarnation, log_list_buckets_iam_implicit_deny,
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, req_info_ref,
};
#[cfg(test)]
use super::storage_api::bucket_usecase::bucket::target::BucketTarget;
@@ -47,7 +47,7 @@ use super::storage_api::bucket_usecase::bucket::{
versioning_sys::BucketVersioningSys,
};
use super::storage_api::bucket_usecase::contract::bucket::{
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
};
use super::storage_api::bucket_usecase::contract::list::{
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _,
@@ -56,9 +56,9 @@ use super::storage_api::bucket_usecase::error::StorageError;
use super::storage_api::bucket_usecase::helper::{OperationHelper, spawn_background_with_context};
use super::storage_api::bucket_usecase::object_utils::to_s3s_xml_etag;
use super::storage_api::bucket_usecase::s3_api::bucket::{
ListObjectVersionsParams, ListObjectsV2Params, build_list_object_versions_output, build_list_objects_output,
build_list_objects_v2_output, encode_list_objects_v2_continuation_token, parse_list_object_versions_params,
parse_list_objects_v2_params, rustfs_owner,
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
parse_list_object_versions_params, parse_list_objects_v2_params, rustfs_owner,
};
use super::storage_api::bucket_usecase::{
get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
@@ -66,7 +66,6 @@ use super::storage_api::bucket_usecase::{
};
#[cfg(feature = "diagnostic-read-version-caller")]
use super::storage_api::diagnostic;
use crate::app::RequestEnvelope;
use crate::app::bucket_list_through as list_through;
use crate::app::object::shared::mark_on_demand_migration_list_local_only;
use crate::app::object_data_cache::invalidate_object_data_cache_bucket_after_delete;
@@ -88,14 +87,12 @@ use http::StatusCode;
use metrics::counter;
use rustfs_config::RUSTFS_REGION;
use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration, PersistedTagging};
use rustfs_gateway_types::dto as gateway_dto;
use rustfs_gateway_types::persistence::{
PersistedBucketEncryptionConfiguration, PersistedNotificationConfiguration,
PersistedObjectLockConfiguration as ObjectLockConfiguration, PersistedPublicAccessBlockConfiguration,
PersistedVersioningConfiguration,
};
use rustfs_gateway_types::persistence::{PersistedLifecycleConfiguration, PersistedReplicationConfiguration};
use rustfs_gateway_types::{BucketName, Timestamp as GatewayTimestamp};
use rustfs_io_metrics::record_s3_op;
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
use rustfs_policy::policy::{
@@ -119,15 +116,15 @@ use s3s::dto::{
GetBucketCorsInput, GetBucketEncryptionInput, GetBucketLifecycleConfigurationInput, GetBucketLocationInput,
GetBucketLocationOutput, GetBucketNotificationConfigurationInput, GetBucketPolicyInput, GetBucketPolicyOutput,
GetBucketPolicyStatusInput, GetBucketPolicyStatusOutput, GetBucketReplicationInput, GetBucketTaggingInput,
GetBucketVersioningInput, GetPublicAccessBlockInput, HeadBucketInput, HeadBucketOutput, LifecycleRule,
ListObjectVersionsInput, ListObjectVersionsOutput, ListObjectsInput, ListObjectsOutput, ListObjectsV2Input,
ListObjectsV2Output, MetadataEntry, NotificationConfiguration, NotificationConfigurationFilter, Object, ObjectStorageClass,
ObjectVersion, ObjectVersionStorageClass, PolicyStatus, PutBucketCorsInput, PutBucketCorsOutput, PutBucketEncryptionInput,
PutBucketEncryptionOutput, PutBucketLifecycleConfigurationInput, PutBucketLifecycleConfigurationOutput,
PutBucketNotificationConfigurationInput, PutBucketNotificationConfigurationOutput, PutBucketPolicyInput,
PutBucketPolicyOutput, PutBucketReplicationInput, PutBucketReplicationOutput, PutBucketTaggingInput, PutBucketTaggingOutput,
PutBucketVersioningInput, PutBucketVersioningOutput, PutPublicAccessBlockInput, PutPublicAccessBlockOutput,
ServerSideEncryption, Timestamp, UserMetadata,
GetBucketVersioningInput, GetPublicAccessBlockInput, HeadBucketInput, HeadBucketOutput, LifecycleRule, ListBucketsInput,
ListBucketsOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ListObjectsInput, ListObjectsOutput,
ListObjectsV2Input, ListObjectsV2Output, MetadataEntry, NotificationConfiguration, NotificationConfigurationFilter, Object,
ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, PolicyStatus, PutBucketCorsInput, PutBucketCorsOutput,
PutBucketEncryptionInput, PutBucketEncryptionOutput, PutBucketLifecycleConfigurationInput,
PutBucketLifecycleConfigurationOutput, PutBucketNotificationConfigurationInput, PutBucketNotificationConfigurationOutput,
PutBucketPolicyInput, PutBucketPolicyOutput, PutBucketReplicationInput, PutBucketReplicationOutput, PutBucketTaggingInput,
PutBucketTaggingOutput, PutBucketVersioningInput, PutBucketVersioningOutput, PutPublicAccessBlockInput,
PutPublicAccessBlockOutput, ServerSideEncryption, Timestamp, UserMetadata,
};
use s3s::region::Region;
use s3s::xml;
@@ -1406,40 +1403,6 @@ fn lifecycle_has_abort_multipart_rules(config: &PersistedLifecycleConfiguration)
.any(|rule| rule.status == "Enabled" && rule.abort_incomplete_multipart_upload.is_some())
}
/// The ListBuckets answer for `bucket_infos`, in the gateway's types.
///
/// A stored name the S3 bucket grammar refuses cannot be written into the answer and is left out.
/// The storage inventory holds such names only for RustFS-internal buckets (the SFTP host-key
/// store); `rustfs/src/protocols/client.rs` applies the same rule to its listings.
fn list_buckets_output(bucket_infos: &[BucketInfo]) -> gateway_dto::ListBucketsOutput {
let buckets = bucket_infos
.iter()
.filter_map(|bucket_info| {
let name = BucketName::new(bucket_info.name.clone()).ok()?;
let created = bucket_info.created.unwrap_or(time::OffsetDateTime::UNIX_EPOCH);
let seconds = created.unix_timestamp();
let creation_date =
GatewayTimestamp::from_secs_nanos(seconds, created.nanosecond()).unwrap_or(GatewayTimestamp::from_secs(seconds));
Some(gateway_dto::Bucket {
name,
creation_date,
..Default::default()
})
})
.collect();
let owner = rustfs_owner();
gateway_dto::ListBucketsOutput {
buckets,
owner: gateway_dto::Owner {
id: owner.id,
display_name: owner.display_name,
},
continuation_token: None,
prefix: None,
}
}
#[derive(Clone, Default)]
pub struct DefaultBucketUsecase {
context: Option<Arc<AppContext>>,
@@ -1776,61 +1739,21 @@ impl DefaultBucketUsecase {
Ok(S3Response::new(GetBucketLocationOutput { location_constraint }))
}
/// Lists the buckets the caller of `envelope` may see, for either edge.
///
/// The access phase must have installed the caller's `ReqInfo` in `envelope`. A caller allowed
/// `s3:ListAllMyBuckets` sees every bucket; any other caller sees the buckets it may list or
/// locate, and is refused when there is none. `_input` is accepted and not read: RustFS answers
/// one unpaginated listing and ignores `prefix`, `max-buckets`, `continuation-token` and
/// `bucket-region`, as it always has.
///
/// # Errors
///
/// `AccessDenied` for an anonymous caller and for one that may see no bucket; `InternalError`
/// when the store or IAM is not initialized or the access phase did not run; otherwise the
/// storage error of the listing.
#[instrument(level = "debug", skip(self, envelope, _input))]
pub async fn execute_list_buckets(
&self,
envelope: RequestEnvelope,
_input: gateway_dto::ListBucketsInput,
) -> rustfs_s3_types::S3Result<gateway_dto::ListBucketsOutput> {
#[instrument(level = "debug", skip(self, req))]
pub async fn execute_list_buckets(&self, req: S3Request<ListBucketsInput>) -> S3Result<S3Response<ListBucketsOutput>> {
let Some(store) = self.object_store() else {
return Err(rustfs_s3_types::S3Error::with_message(
rustfs_s3_types::S3ErrorCode::InternalError,
"Not init",
));
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
if envelope
.credentials()
.is_none_or(|credentials| credentials.access_key().is_empty())
{
return Err(rustfs_s3_types::S3Error::with_message(
rustfs_s3_types::S3ErrorCode::AccessDenied,
"Access Denied",
));
if req.credentials.as_ref().is_none_or(|cred| cred.access_key.is_empty()) {
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
}
let Some(req_info) = envelope.req_info() else {
return Err(rustfs_s3_types::S3Error::with_message(
rustfs_s3_types::S3ErrorCode::InternalError,
"ReqInfo not found in request envelope",
));
};
let iam_authorization = prepare_list_buckets_iam_authorization(ListBucketsAuthzRequest {
req_info,
headers: envelope.headers(),
raw_query: envelope.uri().query(),
remote_addr: envelope.remote_addr(),
client_info: envelope.client_info(),
server_context: envelope.server_context(),
})
.await?;
let iam_authorization = prepare_list_buckets_iam_authorization(&req).await?;
let bucket_infos = if iam_authorization.is_allowed("", S3Action::ListAllMyBucketsAction).await {
store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?
} else {
log_list_buckets_iam_implicit_deny(req_info);
log_list_buckets_iam_implicit_deny(&req)?;
let bucket_infos = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?;
let mut visible_bucket_infos = Vec::new();
for info in bucket_infos {
@@ -1849,7 +1772,7 @@ impl DefaultBucketUsecase {
visible_bucket_infos
};
Ok(list_buckets_output(&bucket_infos))
Ok(S3Response::new(build_list_buckets_output(&bucket_infos)))
}
pub async fn execute_delete_bucket_encryption(
@@ -3989,16 +3912,36 @@ mod tests {
req
}
fn usecase_method_source<'a>(source: &'a str, method: &str) -> &'a str {
let public_start_marker = format!("pub async fn {method}");
let private_start_marker = format!("async fn {method}");
let (start, marker_len) = if let Some(start) = source.find(&public_start_marker) {
(start, public_start_marker.len())
/// Bucket usecase source: this file plus `bucket_usecase/`, where a split moves its methods.
fn bucket_usecase_sources() -> crate::guard_sources::SourceSet {
crate::guard_sources::SourceSet::read("src/app/bucket_usecase.rs", "src/app/bucket_usecase")
}
/// Text after each exact `{marker}(` or `{marker}<` definition, in set order.
fn method_definitions<'a>(source: &'a crate::guard_sources::SourceSet, marker: &str) -> Vec<&'a str> {
source
.files()
.iter()
.flat_map(|file| {
file.text
.match_indices(marker)
.map(move |(start, _)| &file.text[start + marker.len()..])
.filter(|rest| rest.starts_with(['(', '<']))
})
.collect()
}
/// The body of the one definition of `method` across the bucket usecase set, preferring the
/// public form; it ends at the next method of the same indentation or at the end of its file.
fn usecase_method_source<'a>(source: &'a crate::guard_sources::SourceSet, method: &str) -> &'a str {
let public = method_definitions(source, &format!("pub async fn {method}"));
let definitions = if public.is_empty() {
method_definitions(source, &format!("async fn {method}"))
} else {
let start = source.find(&private_start_marker).expect("method should exist");
(start, private_start_marker.len())
public
};
let rest = &source[start + marker_len..];
assert_eq!(definitions.len(), 1, "{method} must be defined exactly once across {}", source.describe());
let rest = definitions[0];
let end = [rest.find("\n pub async fn "), rest.find("\n async fn ")]
.into_iter()
.flatten()
@@ -4009,7 +3952,8 @@ mod tests {
#[test]
fn bucket_create_and_delete_use_admitted_fresh_tasks() {
let source = include_str!("bucket_usecase.rs");
let sources = bucket_usecase_sources();
let source = &sources;
for (method, operation, inner_call) in [
(
"execute_create_bucket",
@@ -4034,7 +3978,8 @@ mod tests {
#[test]
fn bucket_metadata_config_changes_notify_peer_metadata_reload() {
let source = include_str!("bucket_usecase.rs");
let sources = bucket_usecase_sources();
let source = &sources;
for (method, operation, scanner_maintenance_change) in [
("execute_delete_bucket_policy", "delete bucket policy", false),
("execute_put_bucket_policy", "put bucket policy", false),
@@ -5161,60 +5106,13 @@ mod tests {
#[tokio::test]
async fn execute_list_buckets_returns_internal_error_when_store_uninitialized() {
let req = build_request((), Method::GET);
let input = ListBucketsInput::builder().build().unwrap();
let req = build_request(input, Method::GET);
let usecase = DefaultBucketUsecase::without_context();
let err = usecase
.execute_list_buckets(RequestEnvelope::from_s3s(&req), gateway_dto::ListBucketsInput::default())
.await
.unwrap_err();
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
}
fn bucket_info(name: &str, created: Option<time::OffsetDateTime>) -> BucketInfo {
BucketInfo {
name: name.to_owned(),
created,
..Default::default()
}
}
#[test]
fn list_buckets_output_carries_names_dates_and_the_owner() {
let created = time::OffsetDateTime::from_unix_timestamp_nanos(1_700_000_000_123_000_000).expect("valid timestamp");
let output = list_buckets_output(&[bucket_info("alpha", Some(created)), bucket_info("beta", None)]);
let listed: Vec<(&str, i64, u32)> = output
.buckets
.iter()
.map(|bucket| (bucket.name.as_str(), bucket.creation_date.secs(), bucket.creation_date.subsec_nanos()))
.collect();
assert_eq!(listed, [("alpha", 1_700_000_000, 123_000_000), ("beta", 0, 0)]);
assert!(output.owner.id.is_some(), "the listing names its owner");
assert_eq!(output.continuation_token, None);
assert_eq!(output.prefix, None);
}
#[test]
fn list_buckets_output_leaves_out_names_the_bucket_grammar_refuses() {
let output = list_buckets_output(&[
bucket_info("sftp_host_keys", None),
bucket_info("UPPER", None),
bucket_info("ab", None),
bucket_info("kept", None),
]);
let names: Vec<&str> = output.buckets.iter().map(|bucket| bucket.name.as_str()).collect();
assert_eq!(names, ["kept"]);
}
#[test]
fn list_buckets_output_of_no_bucket_is_an_empty_listing_with_an_owner() {
let output = list_buckets_output(&[]);
assert!(output.buckets.is_empty());
assert!(output.owner.id.is_some());
let err = usecase.execute_list_buckets(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[tokio::test]
-1
View File
@@ -26,7 +26,6 @@ mod startup;
pub(crate) use envelope::RequestHead;
pub use envelope::{RequestCredentials, RequestEnvelope};
pub(crate) use envelope::{authorize_owned_internal_object_request, authorize_owned_request};
pub use global::*;
pub use handles::*;
pub use interfaces::*;
+11 -422
View File
@@ -29,19 +29,13 @@ use super::ServerContextSlot;
use crate::app::metadata_route::WebsiteRead;
use crate::app::object::request_body::BodyReadControl;
use crate::app::storage_api::context::{
AuditRequestView, AuthorizationRequest, DenialAuditor, DenialScope, authorize_request_core,
};
use crate::app::storage_api::context::{
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, MultipartStagingOperation,
ObjectTagConditions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectPolicyHeaders,
PostObjectRequestMarker, ReqInfo, RequestContext, RequestedCopySource, RequestedObjectLegalHold,
RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags, StagedMultipartPartAuthorization,
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, ObjectTagConditions,
OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectRequestMarker, ReqInfo, RequestContext,
TableDataPlaneListAccess, TableDataPlanePublicationGuards,
};
use crate::auth::{VerifiedPresignedRequest, VerifiedSigV4Request};
use crate::shared_types::{NativeTlsTransport, RemoteAddr};
use http::{Extensions, HeaderMap, Method, Uri};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_trusted_proxies::ClientInfo;
use std::fmt;
use std::net::SocketAddr;
@@ -51,109 +45,42 @@ use zeroize::Zeroizing;
/// The identity a request was authenticated with.
///
/// The secret key and the session token are bearer material: the type has no
/// `PartialEq`, and its `Debug` output redacts both. An edge that keeps the
/// caller's secret (it verified the signature itself) builds the value with
/// [`Self::without_secret_key`]; the absence is then `None`, never a
/// substitute string a reader could compare, seal or derive with.
/// `PartialEq`, and its `Debug` output redacts both.
#[derive(Clone)]
pub struct RequestCredentials {
access_key: String,
secret_key: Option<Zeroizing<String>>,
secret_key: Zeroizing<String>,
session_token: Option<Zeroizing<String>>,
}
/// An edge handed over an empty secret key. Absence is expressed with
/// [`RequestCredentials::without_secret_key`], so an empty secret is refused
/// rather than read as "absent" or used as a key.
#[derive(Debug)]
pub(crate) struct EmptySecretKey;
impl fmt::Display for EmptySecretKey {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("handed-over secret key is empty")
}
}
impl std::error::Error for EmptySecretKey {}
impl RequestCredentials {
/// Credentials whose secret the edge handed over.
///
/// # Errors
///
/// [`EmptySecretKey`] when `secret_key` is empty.
pub(crate) fn new(access_key: String, secret_key: String, session_token: Option<String>) -> Result<Self, EmptySecretKey> {
if secret_key.is_empty() {
return Err(EmptySecretKey);
}
Ok(Self {
access_key,
secret_key: Some(Zeroizing::new(secret_key)),
session_token: session_token.map(Zeroizing::new),
})
}
/// Credentials whose secret the edge kept.
pub(crate) fn without_secret_key(access_key: String, session_token: Option<String>) -> Self {
pub(crate) fn new(access_key: String, secret_key: String, session_token: Option<String>) -> Self {
Self {
access_key,
secret_key: None,
secret_key: Zeroizing::new(secret_key),
session_token: session_token.map(Zeroizing::new),
}
}
/// The same identity with `session_token` as its token.
#[cfg(test)]
pub(crate) fn with_session_token(mut self, session_token: Option<String>) -> Self {
self.session_token = session_token.map(Zeroizing::new);
self
}
pub fn access_key(&self) -> &str {
&self.access_key
}
/// The caller's secret, or `None` when the edge did not hand it over.
/// A reader must fail closed on `None`. Never log this value or compare it
/// with `==`.
pub fn secret_key(&self) -> Option<&str> {
self.secret_key.as_deref().map(String::as_str)
/// Never log this value or compare it with `==`.
pub fn secret_key(&self) -> &str {
&self.secret_key
}
pub fn session_token(&self) -> Option<&str> {
self.session_token.as_deref().map(String::as_str)
}
/// The handed-over secret, or [`MissingSecretKey`] when the edge kept it.
pub(crate) fn require_secret_key(&self) -> Result<&str, MissingSecretKey> {
self.secret_key().ok_or(MissingSecretKey)
}
/// Whether the handed-over secret equals `stored`, compared in constant
/// time. An absent secret matches nothing, not even an empty `stored`.
pub(crate) fn secret_key_matches(&self, stored: &str) -> bool {
self.secret_key()
.is_some_and(|secret| crate::auth::constant_time_eq(secret, stored))
}
}
/// A reader needed the caller's secret, but the edge did not hand it over.
#[derive(Debug)]
pub(crate) struct MissingSecretKey;
impl fmt::Display for MissingSecretKey {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("the caller's secret key was not handed over for this request")
}
}
impl std::error::Error for MissingSecretKey {}
impl fmt::Debug for RequestCredentials {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RequestCredentials")
.field("access_key", &self.access_key)
.field("secret_key", &self.secret_key.as_ref().map(|_| "<redacted>"))
.field("secret_key", &"<redacted>")
.field("session_token", &self.session_token.as_ref().map(|_| "<redacted>"))
.finish()
}
@@ -201,7 +128,6 @@ pub struct RequestEnvelope {
verified_presigned: Option<VerifiedPresignedRequest>,
website_read: Option<WebsiteRead>,
post_object: Option<PostObjectRequestMarker>,
post_object_policy_headers: Option<PostObjectPolicyHeaders>,
// Attached by the authorization phase (`storage/access.rs`) for the use case.
req_info: Option<ReqInfo>,
bucket_generation: Option<BucketGenerationGuard>,
@@ -212,13 +138,6 @@ pub struct RequestEnvelope {
table_publication_guards: Option<TableDataPlanePublicationGuards>,
table_list_access: Option<TableDataPlaneListAccess>,
object_tag_conditions: Option<ObjectTagConditions>,
multipart_staging_operation: Option<MultipartStagingOperation>,
staged_multipart_part_authorization: Option<StagedMultipartPartAuthorization>,
requested_object_lock_retention: Option<RequestedObjectLockRetention>,
requested_object_lock_retain_until: Option<RequestedObjectLockRetainUntil>,
requested_object_legal_hold: Option<RequestedObjectLegalHold>,
requested_object_tags: Option<RequestedObjectTags>,
requested_copy_source: Option<RequestedCopySource>,
residual_extensions: Extensions,
}
@@ -248,7 +167,6 @@ impl RequestEnvelope {
verified_presigned: extensions.remove(),
website_read: extensions.remove(),
post_object: extensions.remove(),
post_object_policy_headers: extensions.remove(),
req_info: extensions.remove(),
bucket_generation: extensions.remove(),
pending_delete_bucket_generation: extensions.remove(),
@@ -258,13 +176,6 @@ impl RequestEnvelope {
table_publication_guards: extensions.remove(),
table_list_access: extensions.remove(),
object_tag_conditions: extensions.remove(),
multipart_staging_operation: extensions.remove(),
staged_multipart_part_authorization: extensions.remove(),
requested_object_lock_retention: extensions.remove(),
requested_object_lock_retain_until: extensions.remove(),
requested_object_legal_hold: extensions.remove(),
requested_object_tags: extensions.remove(),
requested_copy_source: extensions.remove(),
residual_extensions: extensions,
}
}
@@ -401,10 +312,6 @@ impl RequestEnvelope {
self.table_publication_guards.as_ref()
}
pub(crate) fn install_table_publication_guards(&mut self, guards: TableDataPlanePublicationGuards) {
self.table_publication_guards = Some(guards);
}
pub(crate) fn table_list_access(&self) -> Option<&TableDataPlaneListAccess> {
self.table_list_access.as_ref()
}
@@ -412,337 +319,19 @@ impl RequestEnvelope {
pub(crate) fn object_tag_conditions(&self) -> Option<&ObjectTagConditions> {
self.object_tag_conditions.as_ref()
}
pub(crate) fn install_object_tag_conditions(&mut self, conditions: ObjectTagConditions) {
self.object_tag_conditions = Some(conditions);
}
pub(crate) fn multipart_staging_operation(&self) -> Option<&MultipartStagingOperation> {
self.multipart_staging_operation.as_ref()
}
pub(crate) fn staged_multipart_part_authorization(&self) -> Option<&StagedMultipartPartAuthorization> {
self.staged_multipart_part_authorization.as_ref()
}
pub(crate) fn requested_object_lock_retention(&self) -> Option<&RequestedObjectLockRetention> {
self.requested_object_lock_retention.as_ref()
}
pub(crate) fn requested_object_lock_retain_until(&self) -> Option<&RequestedObjectLockRetainUntil> {
self.requested_object_lock_retain_until.as_ref()
}
pub(crate) fn post_object_policy_headers(&self) -> Option<&PostObjectPolicyHeaders> {
self.post_object_policy_headers.as_ref()
}
pub(crate) fn requested_object_legal_hold(&self) -> Option<&RequestedObjectLegalHold> {
self.requested_object_legal_hold.as_ref()
}
pub(crate) fn requested_object_tags(&self) -> Option<&RequestedObjectTags> {
self.requested_object_tags.as_ref()
}
pub(crate) fn requested_copy_source(&self) -> Option<&RequestedCopySource> {
self.requested_copy_source.as_ref()
}
}
/// The authorization core in `storage` reads requests only through
/// [`AuthorizationRequest`]; this impl hands it the envelope's named fields.
impl AuthorizationRequest for RequestEnvelope {
fn headers(&self) -> &HeaderMap {
self.headers()
}
fn uri(&self) -> &http::Uri {
self.uri()
}
fn remote_addr(&self) -> Option<std::net::SocketAddr> {
self.remote_addr()
}
fn client_info(&self) -> Option<&ClientInfo> {
self.client_info()
}
fn server_context(&self) -> Option<&Arc<ServerContextSlot>> {
self.server_context()
}
fn req_info(&self) -> Option<&ReqInfo> {
self.req_info()
}
fn internal_object_authorization(&self) -> bool {
false
}
fn multipart_staging(&self) -> bool {
self.multipart_staging_operation().is_some()
}
fn staged_multipart_part(&self) -> bool {
self.staged_multipart_part_authorization().is_some()
}
fn requested_retention(&self) -> Option<&RequestedObjectLockRetention> {
self.requested_object_lock_retention()
}
fn requested_retain_until(&self) -> Option<&RequestedObjectLockRetainUntil> {
self.requested_object_lock_retain_until()
}
fn requested_legal_hold(&self) -> Option<&RequestedObjectLegalHold> {
self.requested_object_legal_hold()
}
fn requested_object_tags(&self) -> Option<&RequestedObjectTags> {
self.requested_object_tags()
}
fn requested_copy_source(&self) -> Option<&RequestedCopySource> {
self.requested_copy_source()
}
fn policy_headers(&self) -> &HeaderMap {
self.post_object_policy_headers().map_or(self.headers(), |headers| &headers.0)
}
fn object_tag_conditions(&self) -> Option<&ObjectTagConditions> {
self.object_tag_conditions()
}
fn install_object_tag_conditions(&mut self, conditions: ObjectTagConditions) {
self.install_object_tag_conditions(conditions);
}
fn publication_guards(&self) -> Option<&TableDataPlanePublicationGuards> {
self.table_publication_guards()
}
fn install_publication_guards(&mut self, guards: TableDataPlanePublicationGuards) {
self.install_table_publication_guards(guards);
}
fn audit_denial(&self, auditor: &DenialAuditor, action: &Action, reason: &'static str) {
auditor.audit_owned(&self.audit_view(), action, reason, DenialScope::Request);
}
}
/// Authorizes an owned request through the same IAM, bucket and table policies as the legacy edge.
///
/// Like the legacy `authorize_request`, it first restores the signed `x-amz-meta-*`
/// query parameters into headers, so policy conditions and duplicate rejection see
/// the same request shape on both edges.
///
/// This is only the policy decision. It does **not** perform the composite checks
/// the legacy `S3Access` hooks run around their decision, such as the
/// `s3:BypassGovernanceRetention` authorization for a governance-bypass header or
/// the copy-source read authorization; a caller that needs them runs them itself.
///
/// # Errors
///
/// `InvalidArgument` when the signed query repeats a user metadata key;
/// `AccessDenied` when the decision for `action` depends on a request-body input
/// the envelope does not carry (see [`require_body_policy_inputs`]) or when the
/// policies deny the request.
pub(crate) async fn authorize_owned_request(
req: &mut RequestEnvelope,
action: Action,
location_constraint: Option<String>,
) -> rustfs_s3_types::S3Result<()> {
prepare_owned_authorization(req, action)?;
authorize_request_core(req, action, location_constraint, false)
.await
.map_err(Into::into)
}
/// Authorizes one exact object key after its enclosing table-catalog operation has been authorized.
///
/// It applies the same preparation and has the same limits as [`authorize_owned_request`].
pub(crate) async fn authorize_owned_internal_object_request(
req: &mut RequestEnvelope,
action: Action,
) -> rustfs_s3_types::S3Result<()> {
prepare_owned_authorization(req, action)?;
authorize_request_core(req, action, None, true).await.map_err(Into::into)
}
fn prepare_owned_authorization(req: &mut RequestEnvelope, action: Action) -> rustfs_s3_types::S3Result<()> {
crate::auth::hoist_signed_query_params(&mut req.headers, req.uri.query()).map_err(rustfs_s3_types::S3Error::from)?;
require_body_policy_inputs(req, action)
}
/// Fails closed when the decision for `action` depends on a value the request
/// body supplies but the envelope does not carry it. Policy conditions would
/// otherwise fall back to client headers, which the stored value need not match
/// (header `OFF` with body `ON` would pass a legal-hold Deny).
fn require_body_policy_inputs(req: &RequestEnvelope, action: Action) -> rustfs_s3_types::S3Result<()> {
let missing = match action {
Action::S3Action(S3Action::PutObjectLegalHoldAction) => req.requested_object_legal_hold.is_none(),
Action::S3Action(S3Action::PutObjectTaggingAction | S3Action::PutObjectVersionTaggingAction) => {
req.requested_object_tags.is_none()
}
Action::S3Action(S3Action::PutObjectRetentionAction) => req.requested_object_lock_retention.is_none(),
_ => false,
} || (req.post_object.is_some() && req.post_object_policy_headers.is_none())
|| (req.headers.contains_key(COPY_SOURCE_HEADER) && req.requested_copy_source.is_none());
if missing {
return Err(rustfs_s3_types::s3_error!(
AccessDenied,
"request body policy input is missing for this action"
));
}
Ok(())
}
const COPY_SOURCE_HEADER: &str = "x-amz-copy-source";
impl RequestEnvelope {
/// The request fields an audit entry for this request is built from.
pub(crate) fn audit_view(&self) -> AuditRequestView<'_> {
AuditRequestView::from_facts(
self.headers(),
self.uri(),
self.req_info(),
self.client_info(),
self.remote_addr(),
self.request_context(),
)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn authorization_payloads_move_from_extensions_into_named_fields() {
let mut extensions = Extensions::new();
extensions.insert(MultipartStagingOperation);
extensions.insert(StagedMultipartPartAuthorization);
let mut retention = RequestedObjectLockRetention(Default::default());
retention.0.mode = Some("COMPLIANCE".to_owned().into());
extensions.insert(retention);
let retain_until = time::OffsetDateTime::from_unix_timestamp(2_000_000_000).expect("valid timestamp");
extensions.insert(RequestedObjectLockRetainUntil(retain_until));
let head = || RequestHead {
method: Method::PUT,
uri: "/bucket/key".parse().expect("valid URI"),
headers: HeaderMap::new(),
credentials: None,
region: None,
service: None,
};
let envelope = RequestEnvelope::from_parts(head(), extensions);
let empty = RequestEnvelope::from_parts(head(), Extensions::new());
assert_eq!(
(
envelope.multipart_staging_operation().is_some(),
envelope.staged_multipart_part_authorization().is_some(),
envelope
.requested_object_lock_retention()
.and_then(|value| value.0.mode.as_ref().map(|mode| mode.as_str())),
envelope.requested_object_lock_retain_until().map(|value| value.0),
envelope.residual_extensions().is_empty(),
empty.multipart_staging_operation().is_none(),
empty.staged_multipart_part_authorization().is_none(),
empty.requested_object_lock_retention().is_none(),
empty.requested_object_lock_retain_until().is_none(),
),
(true, true, Some("COMPLIANCE"), Some(retain_until), true, true, true, true, true),
);
}
#[test]
fn request_body_policy_payloads_move_from_extensions_into_named_fields() {
let mut form_headers = HeaderMap::new();
form_headers.insert("x-amz-tagging", http::HeaderValue::from_static("from=form"));
let mut legal_hold = RequestedObjectLegalHold(Default::default());
legal_hold.0.status = Some("ON".to_owned().into());
let mut tags = RequestedObjectTags(Default::default());
tags.0.tag_set.push(Default::default());
tags.0.tag_set[0].key = Some("body".to_owned());
let mut extensions = Extensions::new();
extensions.insert(PostObjectPolicyHeaders(form_headers));
extensions.insert(legal_hold);
extensions.insert(tags);
extensions.insert(RequestedCopySource("source-bucket/source-key".to_owned()));
let head = || RequestHead {
method: Method::PUT,
uri: "/bucket/key".parse().expect("valid URI"),
headers: HeaderMap::new(),
credentials: None,
region: None,
service: None,
};
let envelope = RequestEnvelope::from_parts(head(), extensions);
let empty = RequestEnvelope::from_parts(head(), Extensions::new());
assert_eq!(
(
envelope
.post_object_policy_headers()
.and_then(|headers| headers.0.get("x-amz-tagging"))
.and_then(|value| value.to_str().ok()),
envelope
.requested_object_legal_hold()
.and_then(|hold| hold.0.status.as_ref().map(|status| status.as_str())),
envelope
.requested_object_tags()
.and_then(|tags| tags.0.tag_set.first())
.and_then(|tag| tag.key.as_deref()),
envelope.requested_copy_source().map(|source| source.0.as_str()),
envelope.residual_extensions().is_empty(),
empty.post_object_policy_headers().is_none(),
empty.requested_object_legal_hold().is_none(),
empty.requested_object_tags().is_none(),
empty.requested_copy_source().is_none(),
),
(
Some("from=form"),
Some("ON"),
Some("body"),
Some("source-bucket/source-key"),
true,
true,
true,
true,
true
),
);
}
#[test]
fn an_empty_handed_over_secret_is_refused() {
assert!(RequestCredentials::new("AKIAENVELOPE".to_owned(), String::new(), None).is_err());
}
#[test]
fn an_absent_secret_matches_no_stored_secret() {
let kept = RequestCredentials::without_secret_key("AKIAENVELOPE".to_owned(), None);
assert_eq!(kept.secret_key(), None);
assert!(!kept.secret_key_matches(""), "absent must not equal an empty stored secret");
assert!(!kept.secret_key_matches("stored-secret"));
let handed = RequestCredentials::new("AKIAENVELOPE".to_owned(), "stored-secret".to_owned(), None).expect("secret");
assert!(!handed.secret_key_matches(""));
assert!(!handed.secret_key_matches("other-secret"));
assert!(handed.secret_key_matches("stored-secret"));
}
#[test]
fn a_reader_of_an_absent_secret_fails_closed() {
let kept = RequestCredentials::without_secret_key("AKIAENVELOPE".to_owned(), Some("token".to_owned()));
assert!(kept.require_secret_key().is_err());
let handed = RequestCredentials::new("AKIAENVELOPE".to_owned(), "s".to_owned(), None).expect("secret");
assert_eq!(handed.require_secret_key().ok(), Some("s"));
}
#[test]
fn kept_secret_debug_prints_neither_secret_nor_token() {
let kept = RequestCredentials::without_secret_key("AKIAENVELOPE".to_owned(), Some("envelope-session-token".to_owned()));
let debug = format!("{kept:?}");
assert!(debug.contains("AKIAENVELOPE"), "{debug}");
assert!(!debug.contains("envelope-session-token"), "{debug}");
}
#[test]
fn request_credentials_debug_redacts_secret_and_session_token() {
let credentials = RequestCredentials::new(
"AKIAENVELOPE".to_owned(),
"envelope-secret-value".to_owned(),
Some("envelope-session-token".to_owned()),
)
.expect("non-empty secret");
);
let debug = format!("{credentials:?}");
-3
View File
@@ -36,7 +36,6 @@ pub(crate) mod trailer_adapter;
// from here rather than through `app::context`.
pub(crate) use context::RequestHead;
pub use context::{RequestCredentials, RequestEnvelope};
pub(crate) use context::{authorize_owned_internal_object_request, authorize_owned_request};
#[cfg(test)]
mod capacity_dirty_scope_test;
@@ -49,6 +48,4 @@ pub(crate) mod gating_test_env;
#[cfg(test)]
mod lifecycle_transition_api_test;
#[cfg(test)]
mod owned_authorization_test;
#[cfg(test)]
mod put_prelookup_gating_test;
File diff suppressed because it is too large Load Diff
+88
View File
@@ -0,0 +1,88 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! AbortMultipartUpload.
use super::*;
impl DefaultMultipartUsecase {
#[instrument(level = "debug", skip(self, req))]
pub async fn execute_abort_multipart_upload(
&self,
req: S3Request<AbortMultipartUploadInput>,
) -> S3Result<S3Response<AbortMultipartUploadOutput>> {
reject_presigned_multipart_max_total_object_size_for_other_operation(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedSigV4Request>().is_some(),
)?;
reject_presigned_put_max_content_length_for_other_operation(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
)?;
record_s3_op(S3Operation::AbortMultipartUpload);
let mut opts = ObjectOptions::default();
apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?;
let AbortMultipartUploadInput {
bucket, key, upload_id, ..
} = req.input;
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
let Some(store) = self.object_store() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
// Per the AWS S3 API reference, a malformed upload ID is reported as
// NoSuchUpload (404), the same as an upload that was aborted or
// completed. See multipart_upload_api_error.
match store
.abort_multipart_upload(bucket.as_str(), key.as_str(), upload_id.as_str(), &opts)
.await
{
Ok(_) => {
rustfs_scanner::record_dirty_usage_object_from_producer(
&bucket,
&key,
rustfs_scanner::SegmentInvalidationProducerIdentity::AbortMultipartUpload,
);
Ok(S3Response::new(AbortMultipartUploadOutput { ..Default::default() }))
}
Err(err) => Err(multipart_upload_api_error(err)),
}
}
}
#[cfg(test)]
mod tests {
use super::super::test_support::*;
use super::*;
#[tokio::test]
async fn execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized() {
let input = AbortMultipartUploadInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.upload_id("upload-id".to_string())
.build()
.unwrap();
let req = build_request(input, Method::DELETE);
let before = s3_op_total(S3Operation::AbortMultipartUpload);
let err = make_usecase().execute_abort_multipart_upload(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(s3_op_total(S3Operation::AbortMultipartUpload), before + 1);
}
}
File diff suppressed because it is too large Load Diff
+418
View File
@@ -0,0 +1,418 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! CreateMultipartUpload.
use super::*;
fn create_multipart_upload_metadata(
input_metadata: Option<HashMap<String, String>>,
headers: &HeaderMap,
tagging: Option<String>,
storage_class: Option<&crate::app::storage_api::s3::dto::StorageClass>,
) -> HashMap<String, String> {
let mut metadata = input_metadata.unwrap_or_default();
namespace_reserved_user_metadata(&mut metadata);
extract_metadata_from_mime(headers, &mut metadata);
if let Some(tags) = tagging {
metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags);
}
if let Some(storage_class) = storage_class {
metadata.insert(AMZ_STORAGE_CLASS.to_owned(), storage_class.as_str().to_owned());
}
metadata
}
/// A multipart session advertises disk compression only when the staged-rollout
/// switch (`RUSTFS_COMPRESSION_MULTIPART_ENABLED`) is on, the object key/headers
/// qualify, AND the session is not an SSE-C ciphertext-passthrough replication
/// session, which must preserve source bytes verbatim.
///
/// The rollout switch defaults to off so a rolling upgrade never creates new
/// compressed multipart objects while pre-fix nodes (whose decompressor is not
/// resumable) may still serve reads. Enable it once the fleet has converged on a
/// fixed build; the default flips per the `multipart-compression-default-off-window`
/// entry in docs/architecture/compat-cleanup-register.md.
///
/// Each part is compressed as an independent stream; the GET path decodes across part
/// boundaries (see `ReadTransform::Compressed`), so the session may advertise
/// object-level compression again.
///
/// Unlike single PUT there is no `MIN_DISK_COMPRESSIBLE_SIZE` floor here: the total
/// object size is unknown at CreateMultipartUpload time, so tiny multipart objects pay
/// the (harmless) framing overhead. This is a deliberate trade-off, not a bug.
fn should_advertise_session_compression(multipart_enabled: bool, ciphertext_passthrough: bool, disk_compressible: bool) -> bool {
multipart_enabled && !ciphertext_passthrough && disk_compressible
}
impl DefaultMultipartUsecase {
#[instrument(level = "debug", skip(self, req))]
#[hotpath::measure(impl_type = "MultipartUsecase")]
pub async fn execute_create_multipart_upload(
&self,
req: S3Request<CreateMultipartUploadInput>,
) -> S3Result<S3Response<CreateMultipartUploadOutput>> {
validate_new_object_key(&req.input.key, "CreateMultipartUpload")?;
let multipart_max_total_object_size = parse_presigned_multipart_max_total_object_size(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedSigV4Request>().is_some(),
)?;
reject_presigned_put_max_content_length_for_other_operation(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
)?;
let helper =
OperationHelper::new(&req, EventName::ObjectCreatedCreateMultipartUpload, S3Operation::CreateMultipartUpload)
.suppress_event();
let replication_authorized = replication_request_authorized(&req);
let CreateMultipartUploadInput {
bucket,
key,
tagging,
version_id,
storage_class,
server_side_encryption,
sse_customer_algorithm,
sse_customer_key,
sse_customer_key_md5,
ssekms_key_id,
object_lock_legal_hold_status,
object_lock_mode,
object_lock_retain_until_date,
metadata: input_metadata,
..
} = req.input.clone();
let server_side_encryption = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?);
let ssekms_key_id = ssekms_key_id.or_else(|| {
req.headers
.get("x-amz-server-side-encryption-aws-kms-key-id")
.and_then(|value| value.to_str().ok())
.map(ToOwned::to_owned)
});
// Validate storage class if provided
if let Some(ref storage_class) = storage_class
&& !is_valid_storage_class(storage_class.as_str())
{
return Err(s3_error!(InvalidStorageClass));
}
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
validate_table_catalog_object_mutation(&bucket, &key).await?;
let Some(store) = self.object_store() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
validate_archive_content_encoding(
&key,
req.headers.get("content-type").and_then(|value| value.to_str().ok()),
req.headers.get("content-encoding").and_then(|value| value.to_str().ok()),
)?;
let mut metadata = create_multipart_upload_metadata(input_metadata, &req.headers, tagging, storage_class.as_ref());
if let Some(limit) = multipart_max_total_object_size {
insert_str(&mut metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE, limit.to_string());
}
let has_explicit_object_lock_retention = object_lock_mode.is_some()
|| object_lock_retain_until_date.is_some()
|| has_replication_retention_update(&req.headers, replication_authorized);
let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?;
if let Some(object_lock_metadata) = build_put_like_object_lock_metadata(
&bucket,
&object_lock_config_state,
object_lock_legal_hold_status,
object_lock_mode,
object_lock_retain_until_date,
)? {
metadata.extend(object_lock_metadata);
}
apply_bucket_default_lock_retention(
&bucket,
&object_lock_config_state,
&mut metadata,
has_explicit_object_lock_retention,
)?;
let (header_sse_customer_algorithm, header_sse_customer_key, header_sse_customer_key_md5) =
extract_ssec_params_from_headers(&req.headers)?;
let sse_customer_algorithm = sse_customer_algorithm.or(header_sse_customer_algorithm);
let sse_customer_key = sse_customer_key.or(header_sse_customer_key);
let sse_customer_key_md5 = sse_customer_key_md5.or(header_sse_customer_key_md5);
// The session data key is generated here, so this is where a multipart upload is held
// to the KMS key it names. Parts and completion also authorize their own caller.
let session_principal = SseKmsPrincipal::from_request(&req);
let encryption_request = PrepareEncryptionRequest {
bucket: &bucket,
key: &key,
server_side_encryption,
ssekms_key_id,
ssekms_context: extract_ssekms_context_from_headers(&req.headers)?,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key,
sse_customer_key_md5: sse_customer_key_md5.clone(),
principal: session_principal.as_ref(),
};
// SSE-C ciphertext passthrough: parts are already encrypted, so no
// session DEK is prepared; a session marker tells UploadPart to store
// the ciphertext verbatim instead of recovering encryption material.
let ciphertext_passthrough = replication_authorized
&& get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true")
&& rustfs_utils::http::ssec_transport_to_stored_metadata(&req.headers).is_some();
if ciphertext_passthrough && let Some(metadata_sys) = self.bucket_metadata_sys() {
let check_result = map_quota_check_outcome(
&bucket,
QuotaChecker::new(metadata_sys)
.check_quota(&bucket, QuotaOperation::PutObject, 0)
.await,
)?;
if check_result.quota_limit.is_some() {
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
"SSE-C ciphertext replication is unavailable for quota-enabled buckets".to_string(),
));
}
}
if ciphertext_passthrough {
insert_str(&mut metadata, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, "true".to_string());
}
let prepared_material = if ciphertext_passthrough {
None
} else {
sse_prepare_encryption(encryption_request).await?
};
let (effective_sse, effective_kms_key_id) = match prepared_material {
Some(material) => {
let server_side_encryption = Some(material.server_side_encryption.clone());
let ssekms_key_id = material.response_kms_key_id();
let mut encryption_metadata = encryption_material_to_metadata(&material)?;
if material.key_kind == EncryptionKeyKind::Object {
mark_encrypted_multipart_metadata(&mut encryption_metadata);
}
metadata.extend(encryption_metadata);
(server_side_encryption, ssekms_key_id)
}
None => (None, None),
};
if should_advertise_session_compression(
is_multipart_disk_compression_enabled(),
ciphertext_passthrough,
is_disk_compressible(&req.headers, &key),
) {
rustfs_utils::http::insert_str(
&mut metadata,
rustfs_utils::http::SUFFIX_COMPRESSION,
compression_metadata_value(CompressionAlgorithm::default()),
);
}
let mt2 = metadata.clone();
let mut opts: ObjectOptions =
put_opts_with_replication_authorization(&bucket, &key, version_id, &req.headers, metadata, replication_authorized)
.await
.map_err(ApiError::from)?;
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
let dsc =
must_replicate_object(&bucket, &key, &mt2, "".to_string(), opts.delete_marker_replication_status(), opts.clone())
.await;
if dsc.replicate_any() {
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
insert_str(
&mut opts.user_defined,
SUFFIX_REPLICATION_STATUS,
dsc.pending_status().unwrap_or_default(),
);
}
if !super::super::object::shared::put_like_write_creates_new_version(&opts)
&& super::super::object::shared::object_lock_checks_required_for_state(&object_lock_config_state)
{
let current_opts: ObjectOptions = internal_object_info_lookup_opts(
get_opts(&bucket, &key, opts.version_id.clone(), None, &req.headers)
.await
.map_err(ApiError::from)?,
);
match store.get_object_info(&bucket, &key, &current_opts).await {
Ok(existing_obj_info) => {
validate_existing_object_lock_for_write(&object_lock_config_state, &existing_obj_info, &opts)?
}
Err(err) => {
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
return Err(ApiError::from(err).into());
}
}
}
}
let checksum_type = rustfs_rio::ChecksumType::from_header(&req.headers);
if checksum_type.is(rustfs_rio::ChecksumType::INVALID) {
return Err(s3_error!(InvalidArgument, "Invalid checksum type"));
} else if checksum_type.is_set() && !checksum_type.is(rustfs_rio::ChecksumType::TRAILING) {
opts.want_checksum = Some(rustfs_rio::Checksum {
checksum_type,
..Default::default()
});
}
if effective_sse.is_some() && opts.want_checksum.is_some() {
insert_str(&mut opts.user_defined, SUFFIX_PLAINTEXT_CHECKSUM, "true".to_string());
}
let MultipartUploadResult {
upload_id,
checksum_algo,
checksum_type,
} = store
.new_multipart_upload(&bucket, &key, &opts)
.await
.map_err(ApiError::from)?;
let output = CreateMultipartUploadOutput {
bucket: Some(bucket),
key: Some(key),
upload_id: Some(upload_id),
server_side_encryption: effective_sse,
sse_customer_algorithm,
sse_customer_key_md5,
ssekms_key_id: effective_kms_key_id,
checksum_algorithm: checksum_algo.map(ChecksumAlgorithm::from),
checksum_type: checksum_type.map(ChecksumType::from),
..Default::default()
};
let result = Ok(S3Response::new(output));
let _ = helper.complete(&result);
result
}
}
#[cfg(test)]
mod tests {
use super::super::test_support::*;
use super::*;
#[test]
fn session_compression_is_advertised_only_for_non_passthrough_compressible_uploads() {
// (multipart_enabled, ciphertext_passthrough, disk_compressible, expected)
let cases = [
(true, false, false, false),
(true, false, true, true),
(true, true, false, false),
(true, true, true, false),
// The staged-rollout switch keeps multipart compression dark by
// default regardless of the other gates.
(false, false, true, false),
(false, false, false, false),
(false, true, true, false),
(false, true, false, false),
];
for (multipart_enabled, ciphertext_passthrough, disk_compressible, expected) in cases {
assert_eq!(
should_advertise_session_compression(multipart_enabled, ciphertext_passthrough, disk_compressible),
expected,
"multipart_enabled={multipart_enabled} ciphertext_passthrough={ciphertext_passthrough} disk_compressible={disk_compressible}"
);
}
}
#[tokio::test]
async fn execute_create_multipart_upload_rejects_invalid_storage_class() {
let input = CreateMultipartUploadInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.storage_class(Some(StorageClass::from("invalid".to_string())))
.build()
.unwrap();
let req = build_request(input, Method::POST);
let err = make_usecase().execute_create_multipart_upload(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
}
#[test]
fn create_multipart_upload_metadata_persists_dto_storage_class_without_raw_header() {
let metadata = create_multipart_upload_metadata(
None,
&HeaderMap::new(),
None,
Some(&StorageClass::from_static("REDUCED_REDUNDANCY")),
);
assert_eq!(metadata.get(AMZ_STORAGE_CLASS), Some(&"REDUCED_REDUNDANCY".to_string()));
}
#[test]
fn create_multipart_upload_metadata_keeps_user_and_system_namespaces_separate() {
let mut headers = HeaderMap::new();
headers.insert("content-type", HeaderValue::from_static("application/octet-stream"));
headers.insert(AMZ_STORAGE_CLASS, HeaderValue::from_static("STANDARD"));
let input_metadata = HashMap::from([
("content-type".to_string(), "user-content-type".to_string()),
(AMZ_STORAGE_CLASS.to_string(), "user-storage-class".to_string()),
]);
let metadata = create_multipart_upload_metadata(
Some(input_metadata),
&headers,
Some("project=rustfs".to_string()),
Some(&StorageClass::from_static("REDUCED_REDUNDANCY")),
);
assert_eq!(metadata.get("content-type"), Some(&"application/octet-stream".to_string()));
assert_eq!(metadata.get("x-amz-meta-content-type"), Some(&"user-content-type".to_string()));
assert_eq!(metadata.get(AMZ_STORAGE_CLASS), Some(&"REDUCED_REDUNDANCY".to_string()));
assert_eq!(metadata.get("x-amz-meta-x-amz-storage-class"), Some(&"user-storage-class".to_string()));
assert_eq!(metadata.get(AMZ_OBJECT_TAGGING), Some(&"project=rustfs".to_string()));
}
#[tokio::test]
#[serial_test::serial]
async fn create_multipart_rejects_ciphertext_replication_before_parts_are_staged() {
let (_store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-multipart-quota", 4096).await;
let usecase = DefaultMultipartUsecase::from_global();
let input = CreateMultipartUploadInput::builder()
.bucket(bucket)
.key("object".to_string())
.build()
.expect("create multipart request should build");
let mut request = build_request(input, Method::POST);
insert_header(&mut request.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
request
.headers
.insert(rustfs_utils::http::REPLICATION_SSEC_ALGORITHM_HEADER, HeaderValue::from_static("AES256"));
request.extensions.insert(crate::storage::access::ReqInfo {
replication_request_authorized: true,
..Default::default()
});
let err = usecase
.execute_create_multipart_upload(request)
.await
.expect_err("quota-enabled ciphertext multipart replication should fail before upload creation");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
}
}
+235
View File
@@ -0,0 +1,235 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! ListMultipartUploads and ListParts.
use super::*;
impl DefaultMultipartUsecase {
pub async fn execute_list_multipart_uploads(
&self,
req: S3Request<ListMultipartUploadsInput>,
) -> S3Result<S3Response<ListMultipartUploadsOutput>> {
let table_list_access = req.extensions.get::<TableDataPlaneListAccess>().cloned();
reject_presigned_multipart_max_total_object_size_for_other_operation(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedSigV4Request>().is_some(),
)?;
reject_presigned_put_max_content_length_for_other_operation(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
)?;
let mut opts = ObjectOptions::default();
apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?;
let ListMultipartUploadsInput {
bucket,
prefix,
delimiter,
key_marker,
upload_id_marker,
max_uploads,
encoding_type,
..
} = req.input;
let ListMultipartUploadsParams {
prefix,
key_marker,
max_uploads,
} = parse_list_multipart_uploads_params(prefix, key_marker, max_uploads)?;
let Some(store) = self.object_store() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
// `apply_bucket_generation_guard` tolerates a missing guard (only the S3
// access layer installs one), so resolve the current generation rather
// than failing the request. Listing is filtered by this value, so a
// stale one simply hides foreign-incarnation uploads, as intended.
let expected_incarnation_id = match opts.expected_bucket_incarnation_id {
Some(incarnation_id) => incarnation_id,
None => store.bucket_incarnation_id_from_disk(&bucket).await.map_err(ApiError::from)?,
};
let result = match table_list_access.as_ref() {
Some(access) => {
table_list_isolation::list_multipart_uploads(
store.clone(),
access,
table_list_isolation::ListMultipartUploadsRequest {
bucket: &bucket,
prefix: &prefix,
key_marker: key_marker.as_deref(),
upload_id_marker: upload_id_marker.as_deref(),
delimiter: delimiter.as_deref(),
max_uploads,
expected_incarnation_id,
},
)
.await?
}
None => store
.list_multipart_uploads_for_bucket_incarnation(
&bucket,
&prefix,
key_marker,
upload_id_marker,
delimiter,
max_uploads,
expected_incarnation_id,
)
.await
.map_err(ApiError::from)?,
};
Ok(S3Response::new(build_list_multipart_uploads_output(
bucket,
prefix,
result,
encoding_type.as_ref(),
)))
}
pub async fn execute_list_parts(&self, req: S3Request<ListPartsInput>) -> S3Result<S3Response<ListPartsOutput>> {
reject_presigned_multipart_max_total_object_size_for_other_operation(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedSigV4Request>().is_some(),
)?;
reject_presigned_put_max_content_length_for_other_operation(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
)?;
let mut opts = ObjectOptions::default();
apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?;
let ListPartsInput {
bucket,
key,
upload_id,
part_number_marker,
max_parts,
..
} = req.input;
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
let params = parse_list_parts_params(part_number_marker, max_parts)?;
let Some(store) = self.object_store() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let res = store
.list_object_parts(&bucket, &key, &upload_id, params.part_number_marker, params.max_parts, &opts)
.await
.map_err(multipart_upload_api_error)?;
Ok(S3Response::new(build_list_parts_output(res)))
}
}
#[cfg(test)]
mod tests {
use super::super::test_support::*;
use super::*;
#[tokio::test]
async fn execute_list_multipart_uploads_returns_internal_error_when_store_uninitialized() {
let input = ListMultipartUploadsInput::builder()
.bucket("bucket".to_string())
.build()
.unwrap();
let req = build_request(input, Method::GET);
let err = make_usecase().execute_list_multipart_uploads(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[tokio::test]
async fn execute_list_multipart_uploads_rejects_invalid_key_marker_before_store_lookup() {
let input = ListMultipartUploadsInput::builder()
.bucket("bucket".to_string())
.prefix(Some("prefix/".to_string()))
.key_marker(Some("other/key".to_string()))
.build()
.unwrap();
let req = build_request(input, Method::GET);
let err = make_usecase().execute_list_multipart_uploads(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(err.message(), Some("Invalid key marker"));
}
#[tokio::test]
async fn execute_list_multipart_uploads_rejects_invalid_max_uploads_before_store_lookup() {
let input = ListMultipartUploadsInput::builder()
.bucket("bucket".to_string())
.max_uploads(Some(0))
.build()
.unwrap();
let req = build_request(input, Method::GET);
let expected = "max-uploads must be between 1 and 1000";
let err = make_usecase().execute_list_multipart_uploads(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(err.message(), Some(expected));
}
#[tokio::test]
async fn execute_list_parts_returns_internal_error_when_store_uninitialized() {
let input = ListPartsInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.upload_id("upload-id".to_string())
.build()
.unwrap();
let req = build_request(input, Method::GET);
let err = make_usecase().execute_list_parts(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[tokio::test]
async fn execute_list_parts_rejects_negative_part_number_marker_before_store_lookup() {
let input = ListPartsInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.upload_id("upload-id".to_string())
.part_number_marker(Some(-1))
.build()
.unwrap();
let req = build_request(input, Method::GET);
let err = make_usecase().execute_list_parts(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(err.message(), Some("part-number-marker must be non-negative"));
}
#[tokio::test]
async fn execute_list_parts_rejects_invalid_max_parts_before_store_lookup() {
let input = ListPartsInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.upload_id("upload-id".to_string())
.max_parts(Some(1001))
.build()
.unwrap();
let req = build_request(input, Method::GET);
let err = make_usecase().execute_list_parts(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(err.message(), Some("max-parts must be between 1 and 1000"));
}
}
@@ -0,0 +1,54 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Fixtures and test-only imports shared by the multipart use-case tests.
use super::*;
pub(super) use crate::app::storage_api::s3::dto::CompletedMultipartUpload;
pub(super) use crate::app::storage_api::s3::dto::StorageClass;
pub(super) use http::Extensions;
pub(super) use http::Method;
pub(super) use rustfs_filemeta::ObjectPartInfo;
pub(super) use rustfs_utils::http::AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER;
pub(super) use rustfs_utils::http::AMZ_OBJECT_LOCK_MODE_LOWER;
pub(super) use rustfs_utils::http::AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER;
pub(super) use std::io::Cursor;
pub(super) use temp_env::async_with_vars;
pub(super) use tokio::io::AsyncReadExt;
pub(super) fn s3_op_total(op: S3Operation) -> u64 {
rustfs_io_metrics::s3_op_metrics_snapshot()
.into_iter()
.find(|snapshot| snapshot.op == op.as_str())
.map(|snapshot| snapshot.total)
.unwrap_or_default()
}
pub(super) fn build_request<T>(input: T, method: Method) -> S3Request<T> {
S3Request {
input,
method,
uri: Uri::from_static("/"),
headers: HeaderMap::new(),
extensions: Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
}
pub(super) fn make_usecase() -> DefaultMultipartUsecase {
DefaultMultipartUsecase::without_context()
}
@@ -0,0 +1,823 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! UploadPart.
use super::*;
fn resolve_upload_part_size(content_length: Option<i64>, body: Option<&StreamingBlob>) -> rustfs_s3_types::S3Result<Option<i64>> {
if let Some(length) = content_length {
return Ok(Some(length));
}
body.and_then(|body| body.remaining_length().exact())
.map(i64::try_from)
.transpose()
.map_err(|_| rustfs_s3_types::s3_error!(UnexpectedContent))
}
fn require_upload_part_size(size: Option<i64>, capped: bool) -> rustfs_s3_types::S3Result<i64> {
match size {
Some(size) if size >= 0 => Ok(size),
Some(_) => Err(rustfs_s3_types::s3_error!(UnexpectedContent)),
None if capped => Err(rustfs_s3_types::s3_error!(UnexpectedContent)),
None => Err(rustfs_s3_types::S3Error::new(rustfs_s3_types::S3ErrorCode::MissingContentLength)),
}
}
fn upload_part_body_read_timeout(configured: Duration, capped: bool) -> Duration {
if capped {
configured.max(Duration::from_secs(rustfs_config::DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT))
} else {
configured
}
}
/// Plaintext length of one SSE-C passthrough part, declared by the sender on
/// UploadPart (backlog#2363).
fn passthrough_part_actual_size(headers: &HeaderMap) -> Option<i64> {
get_header(headers, rustfs_utils::http::SUFFIX_REPLICATION_PART_ACTUAL_SIZE)?
.parse::<i64>()
.ok()
.filter(|size| *size > 0)
}
// Hoisted values are signed query parameters, not HTTP headers. Normalize only
// after access control has verified the presigned request.
fn normalize_presigned_part_checksums(req: &mut S3Request<UploadPartInput>) -> S3Result<()> {
if req.extensions.get::<VerifiedPresignedRequest>().is_none() {
return Ok(());
}
let Some(query) = req.uri.query() else {
return Ok(());
};
let mut seen = std::collections::HashSet::new();
for (name, value) in url::form_urlencoded::parse(query.as_bytes()) {
let field = match name.as_ref() {
"x-amz-checksum-crc32" => Some(&mut req.input.checksum_crc32),
"x-amz-checksum-crc32c" => Some(&mut req.input.checksum_crc32c),
"x-amz-checksum-crc64nvme" => Some(&mut req.input.checksum_crc64nvme),
"x-amz-checksum-sha1" => Some(&mut req.input.checksum_sha1),
"x-amz-checksum-sha256" => Some(&mut req.input.checksum_sha256),
"x-amz-sdk-checksum-algorithm" => None,
_ => continue,
};
if !seen.insert(name.clone()) {
return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "Duplicate checksum query parameter"));
}
if let Some(header) = req.headers.get(name.as_ref())
&& header.as_bytes() != value.as_bytes()
{
return Err(S3Error::with_message(
S3ErrorCode::InvalidRequest,
"Conflicting checksum header and query parameter",
));
}
let header = http::HeaderValue::from_str(&value)
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid checksum query parameter"))?;
req.headers.insert(
http::header::HeaderName::from_bytes(name.as_bytes())
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid checksum query parameter"))?,
header,
);
if let Some(field) = field {
*field = Some(value.into_owned());
} else {
req.input.checksum_algorithm = Some(ChecksumAlgorithm::from(value.into_owned()));
}
}
Ok(())
}
impl DefaultMultipartUsecase {
#[instrument(level = "debug", skip(self, req))]
#[hotpath::measure(impl_type = "MultipartUsecase")]
pub async fn execute_upload_part(&self, mut req: S3Request<UploadPartInput>) -> S3Result<S3Response<UploadPartOutput>> {
reject_presigned_multipart_max_total_object_size_for_other_operation(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedSigV4Request>().is_some(),
)?;
reject_presigned_put_max_content_length_for_other_operation(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
)?;
normalize_presigned_part_checksums(&mut req)?;
let mut opts = ObjectOptions::default();
apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?;
let part_principal = SseKmsPrincipal::from_request(&req);
let input = req.input;
let UploadPartInput {
body,
bucket,
key,
upload_id,
part_number,
content_length,
sse_customer_algorithm,
sse_customer_key,
sse_customer_key_md5,
// content_md5,
..
} = input;
let part_id = parse_upload_part_number(part_number)?;
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
validate_table_catalog_object_mutation(&bucket, &key).await?;
let size = resolve_upload_part_size(content_length, body.as_ref())?;
if let Some(size) = size {
reject_oversize_single_upload(size)?;
}
let body_stream = body.ok_or_else(|| s3_error!(IncompleteBody))?;
let Some(store) = self.object_store() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let fi = store
.get_multipart_info(&bucket, &key, &upload_id, &opts)
.await
.map_err(multipart_upload_api_error)?;
authorize_sse_kms_multipart_write(part_principal.as_ref(), &fi.user_defined).await?;
let max_total_object_size = multipart_max_total_object_size(&fi.user_defined)?;
let mut size = require_upload_part_size(size, max_total_object_size.is_some())?;
if let Some(limit) = max_total_object_size
&& u64::try_from(size).is_ok_and(|size| size > limit)
{
return Err(S3Error::new(S3ErrorCode::EntityTooLarge));
}
let admission_wait_started = rustfs_io_metrics::put_stage_timer();
let admission = self.concurrency_manager().admit_multipart_part(size).await;
rustfs_io_metrics::record_put_object_stage_duration_from("multipart_admission_wait", admission_wait_started);
let upload_part_admission = match admission
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "foreground write admission closed"))?
{
ForegroundWriteAdmission::Disabled => None,
ForegroundWriteAdmission::Admitted(permit) => {
counter!("rustfs.upload_part.foreground_admission.total", "result" => "admitted").increment(1);
Some(permit)
}
ForegroundWriteAdmission::Rejected => {
counter!("rustfs.upload_part.foreground_admission.total", "result" => "rejected").increment(1);
return Err(S3Error::with_message(
S3ErrorCode::SlowDown,
"foreground write concurrency limit reached, please reduce your request rate",
));
}
};
let request_id = req
.extensions
.get::<super::super::storage_api::multipart_usecase::request_context::RequestContext>()
.map(|ctx| ctx.request_id.as_str())
.unwrap_or_default();
let timeout = upload_part_body_read_timeout(put_object_body_read_timeout(), max_total_object_size.is_some());
let (body_stream, progress_control) =
if let Some(control) = req.extensions.get::<super::super::object::request_body::BodyReadControl>() {
control.activate_upload(timeout, &bucket, &key, request_id, size.max(0) as u64);
(body_stream, control.clone())
} else {
super::super::object::request_body::guard_upload_progress(
body_stream,
timeout,
&bucket,
&key,
request_id,
size.max(0) as u64,
)
};
let ingress_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(std::time::Instant::now);
// Apply adaptive buffer sizing based on part size for optimal streaming performance.
// Uses workload profile configuration (enabled by default) to select appropriate buffer size.
// Buffer sizes range from 32KB to 4MB depending on part size and configured workload profile.
let buffer_size = get_buffer_size_opt_in(size);
let body = tokio::io::BufReader::with_capacity(
buffer_size,
StreamReader::new(body_stream.map(|f| f.map_err(s3s_body_error_to_io))),
);
// An SSE-C passthrough session stores ciphertext parts verbatim: a
// compression key restored on the session describes those stored
// bytes and must not add a second compression layer, and each part's
// plaintext length comes from the sender (backlog#2363).
let preserve_ciphertext = contains_key_str(&fi.user_defined, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT);
let is_disk_compressed = !preserve_ciphertext
&& rustfs_utils::http::contains_key_str(&fi.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
let actual_size = if preserve_ciphertext {
passthrough_part_actual_size(&req.headers).unwrap_or(size)
} else {
size
};
let mut md5hex = if let Some(base64_md5) = input.content_md5 {
let md5 = base64_simd::STANDARD
.decode_to_vec(base64_md5.as_bytes())
.map_err(|e| ApiError::from(StorageError::other(format!("Invalid content MD5: {e}"))))?;
Some(hex_simd::encode_to_string(&md5, hex_simd::AsciiCase::Lower))
} else {
None
};
let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query());
let mut write_plan = WritePlan::new();
let mut reader = if is_disk_compressed {
let algorithm = CompressionAlgorithm::default();
let mut hrd = HashReader::from_stream(body, size, actual_size, md5hex.take(), sha256hex.take(), false)
.map_err(ApiError::from)?;
if let Err(err) = hrd.add_checksum(&req.headers, trailer_source(req.trailing_headers.clone()), false) {
return Err(ApiError::from(err).into());
}
size = HashReader::SIZE_PRESERVE_LAYER;
write_plan = write_plan.with_compression(algorithm);
hrd
} else {
HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
};
if let Err(err) = reader.add_checksum(&req.headers, trailer_source(req.trailing_headers.clone()), size < 0) {
return Err(ApiError::from(err).into());
}
opts.want_checksum = reader.checksum();
// An SSE-C passthrough session stores ciphertext parts verbatim: no
// material recovery, no validation against the (absent) customer key.
let has_ssec = !preserve_ciphertext
&& fi
.user_defined
.contains_key("x-amz-server-side-encryption-customer-algorithm");
let (server_side_encryption, ssekms_key_id) = if has_ssec || preserve_ciphertext {
(None, None)
} else {
let sse = fi
.user_defined
.get("x-amz-server-side-encryption")
.map(|s| {
ServerSideEncryption::from_str(s)
.map_err(|e| ApiError::from(StorageError::other(format!("Invalid server-side encryption: {e}"))))
})
.transpose()?;
let key_id = match sse.as_ref() {
Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => fi
.user_defined
.get("x-amz-server-side-encryption-aws-kms-key-id")
.map(|s| s.to_string()),
_ => None,
};
(sse, key_id)
};
if !preserve_ciphertext {
EncryptionRequest {
bucket: &bucket,
key: &key,
server_side_encryption: server_side_encryption.clone(),
ssekms_key_id: ssekms_key_id.clone(),
ssekms_context: None,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key: sse_customer_key.clone(),
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: actual_size,
principal: None,
}
.validate_multipart_ssec(&fi.user_defined)?;
}
let (requested_sse, requested_kms_key_id) = if has_ssec {
let ssec_material = sse_decryption(DecryptionRequest {
bucket: &bucket,
key: &key,
metadata: &fi.user_defined,
sse_customer_key: sse_customer_key.as_ref(),
sse_customer_key_md5: sse_customer_key_md5.as_ref(),
principal: None,
})
.await?
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
let ssec_write = match ssec_material.key_kind {
EncryptionKeyKind::Object => WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32),
EncryptionKeyKind::Direct => {
WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id)
}
};
write_plan = write_plan.with_encryption(ssec_write);
(Some(ssec_material.server_side_encryption), ssec_material.kms_key_id)
} else if let Some(server_side_encryption) = server_side_encryption {
// Reuse the stored envelope after authorizing this caller against its
// pinned destination key above.
let managed_material = sse_decryption(DecryptionRequest {
bucket: &bucket,
key: &key,
metadata: &fi.user_defined,
sse_customer_key: None,
sse_customer_key_md5: None,
principal: None,
})
.await?
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
let managed_write = match managed_material.key_kind {
EncryptionKeyKind::Object => WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32),
EncryptionKeyKind::Direct => {
WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_id)
}
};
write_plan = write_plan.with_encryption(managed_write);
(Some(server_side_encryption), ssekms_key_id)
} else {
(None, None)
};
reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
{
use rustfs_rio::HashReaderMut;
let inner = reader.take_inner();
reader.inner =
rustfs_rio::boxed_reader(super::super::object::request_body::DemandReader::new(inner, progress_control));
}
let mut reader = PutObjReader::new(reader);
if let Some(stage_start) = ingress_stage_start {
rustfs_io_metrics::record_put_object_stage_duration(
"multipart_ingress_prepare",
stage_start.elapsed().as_secs_f64() * 1000.0,
);
}
let _upload_part_admission = upload_part_admission;
let store_write_started = rustfs_io_metrics::put_stage_timer();
let result = store
.put_object_part(&bucket, &key, &upload_id, part_id, &mut reader, &opts)
.await;
rustfs_io_metrics::record_put_object_stage_duration_from("multipart_store_write", store_write_started);
drop(_upload_part_admission);
let info = result.map_err(ApiError::from)?;
let mut checksum_crc32 = input.checksum_crc32;
let mut checksum_crc32c = input.checksum_crc32c;
let mut checksum_sha1 = input.checksum_sha1;
let mut checksum_sha256 = input.checksum_sha256;
let mut checksum_crc64nvme = input.checksum_crc64nvme;
if let Some(alg) = &input.checksum_algorithm
&& let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| {
let key = match alg.as_str() {
ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(),
ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(),
ChecksumAlgorithm::SHA1 => rustfs_rio::ChecksumType::SHA1.key(),
ChecksumAlgorithm::SHA256 => rustfs_rio::ChecksumType::SHA256.key(),
ChecksumAlgorithm::CRC64NVME => rustfs_rio::ChecksumType::CRC64_NVME.key(),
_ => return None,
};
trailer.read(|headers| {
headers
.get(key.unwrap_or_default())
.and_then(|value| value.to_str().ok().map(|s| s.to_string()))
})
})
{
match alg.as_str() {
ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str,
ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str,
ChecksumAlgorithm::SHA1 => checksum_sha1 = checksum_str,
ChecksumAlgorithm::SHA256 => checksum_sha256 = checksum_str,
ChecksumAlgorithm::CRC64NVME => checksum_crc64nvme = checksum_str,
_ => (),
}
}
// XXHash3/64/128 and SHA-512 have no typed UploadPartOutput field; echo the
// server-computed part checksum as a raw response header (#1261).
let upload_part_extra_checksum_headers = crate::app::object_usecase::additional_checksum_echo_pairs(&opts.want_checksum);
let output = UploadPartOutput {
server_side_encryption: requested_sse,
ssekms_key_id: requested_kms_key_id,
sse_customer_algorithm,
sse_customer_key_md5,
checksum_crc32,
checksum_crc32c,
checksum_sha1,
checksum_sha256,
checksum_crc64nvme,
e_tag: info.etag.map(|etag| to_s3s_etag(&etag)),
..Default::default()
};
let mut response = S3Response::new(output);
crate::app::object_usecase::inject_additional_checksum_headers(
&mut response.headers,
&upload_part_extra_checksum_headers,
);
Ok(response)
}
}
#[cfg(test)]
mod tests {
use super::super::test_support::*;
use super::*;
fn presigned_checksum_request(query: &str) -> S3Request<UploadPartInput> {
let mut req = build_request(UploadPartInput::default(), Method::PUT);
req.uri = format!("/bucket/object?{query}").parse().unwrap();
req.extensions.insert(VerifiedPresignedRequest);
req
}
#[tokio::test]
async fn normalize_presigned_part_checksums_validates_body() {
// SHA256("abc"), including percent-encoded base64 punctuation.
let checksum = "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=";
let query =
"x-amz-checksum-sha256=ungWv48Bz%2BpBQUDeXa4iI7ADYaOWF3qctBD%2FYfIAFa0%3D&x-amz-sdk-checksum-algorithm=SHA256";
for payload in [b"abc", b"abd"] {
let mut req = presigned_checksum_request(query);
normalize_presigned_part_checksums(&mut req).unwrap();
assert_eq!(req.input.checksum_sha256.as_deref(), Some(checksum));
assert_eq!(
req.input.checksum_algorithm.as_ref().map(|algorithm| algorithm.as_str()),
Some(ChecksumAlgorithm::SHA256)
);
let mut reader = HashReader::from_stream(Cursor::new(payload), 3, 3, None, None, false).unwrap();
reader.add_checksum(&req.headers, None, false).unwrap();
assert_eq!(reader.content_crc_type(), Some(rustfs_rio::ChecksumType::SHA256));
let mut bytes = Vec::new();
let result = reader.read_to_end(&mut bytes).await;
if payload == b"abc" {
result.unwrap();
assert_eq!(bytes, payload);
assert_eq!(reader.content_crc().get("SHA256").map(String::as_str), Some(checksum));
} else {
let err = S3Error::from(ApiError::from(result.unwrap_err()));
assert_eq!(*err.code(), S3ErrorCode::BadDigest);
}
}
}
#[tokio::test]
async fn normalize_presigned_part_checksums_accepts_sdk_crc32_default() {
let mut req = presigned_checksum_request("x-amz-checksum-crc32=y%2FQ5Jg%3D%3D&x-amz-sdk-checksum-algorithm=CRC32");
normalize_presigned_part_checksums(&mut req).unwrap();
assert_eq!(req.input.checksum_crc32.as_deref(), Some("y/Q5Jg=="));
let mut reader = HashReader::from_stream(Cursor::new(b"123456789"), 9, 9, None, None, false).unwrap();
reader.add_checksum(&req.headers, None, false).unwrap();
let mut bytes = Vec::new();
reader.read_to_end(&mut bytes).await.unwrap();
assert_eq!(bytes, b"123456789");
assert_eq!(reader.content_crc().get("CRC32").map(String::as_str), Some("y/Q5Jg=="));
}
#[test]
fn normalize_presigned_part_checksums_rejects_ambiguous_values() {
for query in [
"x-amz-checksum-sha256=one&x-amz-checksum-sha256=two",
"x-amz-sdk-checksum-algorithm=SHA256&x-amz-sdk-checksum-algorithm=CRC32",
] {
let mut req = presigned_checksum_request(query);
assert_eq!(
*normalize_presigned_part_checksums(&mut req).unwrap_err().code(),
S3ErrorCode::InvalidRequest
);
}
let mut req = presigned_checksum_request("x-amz-checksum-sha256=one");
req.headers.insert("x-amz-checksum-sha256", HeaderValue::from_static("two"));
assert_eq!(
*normalize_presigned_part_checksums(&mut req).unwrap_err().code(),
S3ErrorCode::InvalidRequest
);
}
#[test]
fn normalize_presigned_part_checksums_preserves_headers_and_rejects_invalid_values() {
let mut req = presigned_checksum_request("partNumber=1&uploadId=test");
req.input.checksum_sha256 = Some("existing".to_owned());
req.headers
.insert("x-amz-checksum-sha256", HeaderValue::from_static("existing"));
normalize_presigned_part_checksums(&mut req).unwrap();
assert_eq!(req.input.checksum_sha256.as_deref(), Some("existing"));
assert_eq!(req.headers["x-amz-checksum-sha256"], "existing");
let mut req = presigned_checksum_request("x-amz-checksum-sha256=%0D%0A");
assert_eq!(
*normalize_presigned_part_checksums(&mut req).unwrap_err().code(),
S3ErrorCode::InvalidArgument
);
}
#[test]
fn normalize_presigned_part_checksums_requires_verified_presign() {
let mut req = presigned_checksum_request("x-amz-checksum-sha256=one");
req.extensions.remove::<VerifiedPresignedRequest>();
normalize_presigned_part_checksums(&mut req).unwrap();
assert!(req.input.checksum_sha256.is_none());
assert!(req.headers.is_empty());
}
#[test]
fn resolve_upload_part_size_uses_normalized_logical_length() {
let result: rustfs_s3_types::S3Result<Option<i64>> = resolve_upload_part_size(Some(5242880), None);
assert_eq!(result.expect("DTO length"), Some(5242880));
assert_eq!(resolve_upload_part_size(None, None).expect("unknown length"), None);
let body = StreamingBlob::from(Bytes::from_static(b"abc"));
assert_eq!(resolve_upload_part_size(None, Some(&body)).expect("exact bytes"), Some(3));
assert_eq!(resolve_upload_part_size(Some(0), Some(&body)).expect("explicit length wins"), Some(0));
let body = StreamingBlob::wrap(futures::stream::iter([Ok::<_, std::io::Error>(Bytes::from_static(b"abc"))]));
assert_eq!(futures::Stream::size_hint(&body), (1, Some(1)), "the stream knows its item count");
assert_eq!(resolve_upload_part_size(None, Some(&body)).expect("unknown byte length"), None);
}
#[test]
fn upload_part_length_contract_rejects_unknown_and_all_negative_lengths() {
let result: rustfs_s3_types::S3Result<i64> = require_upload_part_size(None, false);
assert_eq!(
result.expect_err("ordinary unknown").code(),
&rustfs_s3_types::S3ErrorCode::MissingContentLength
);
assert_eq!(
require_upload_part_size(None, true).expect_err("capped unknown").code(),
&rustfs_s3_types::S3ErrorCode::UnexpectedContent
);
for capped in [false, true] {
for size in [-1, -2, i64::MIN] {
assert_eq!(
require_upload_part_size(Some(size), capped).expect_err("negative").code(),
&rustfs_s3_types::S3ErrorCode::UnexpectedContent
);
}
assert_eq!(require_upload_part_size(Some(0), capped).expect("zero is valid"), 0);
}
}
#[test]
fn upload_part_timeout_policy_preserves_disabled_and_capped_floor() {
for seconds in [0, 1, 299, 300, 601] {
let configured = Duration::from_secs(seconds);
assert_eq!(upload_part_body_read_timeout(configured, false), configured);
assert_eq!(upload_part_body_read_timeout(configured, true), Duration::from_secs(seconds.max(300)));
}
}
#[tokio::test]
async fn execute_upload_part_rejects_missing_body() {
let input = UploadPartInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.upload_id("upload-id".to_string())
.part_number(1)
.build()
.unwrap();
let req = build_request(input, Method::PUT);
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::IncompleteBody);
}
/// issue #7596: a part whose declared length exceeds the 5 GiB
/// single-request ceiling is rejected before the body is polled or the
/// store is consulted. Exact-cap and zero-length parts pass admission.
#[tokio::test]
async fn execute_upload_part_rejects_oversize_declared_part_before_reading_the_body() {
let ceiling = i64::try_from(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE).expect("ceiling fits i64");
for (declared, expect_too_large) in [(ceiling + 1, true), (ceiling, false), (0, false)] {
let (body, polls) = crate::app::object::PollCountingBody::streaming_blob();
let input = UploadPartInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.upload_id("upload-id".to_string())
.part_number(1)
.body(Some(body))
.content_length(Some(declared))
.build()
.unwrap();
let req = build_request(input, Method::PUT);
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
if expect_too_large {
assert_eq!(err.code(), &S3ErrorCode::EntityTooLarge, "declared {declared}");
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
} else {
assert_ne!(err.code(), &S3ErrorCode::EntityTooLarge, "declared {declared} must pass admission");
}
}
}
#[tokio::test]
async fn execute_upload_part_rejects_invalid_part_number_before_body_lookup() {
for part_number in [-1, 0, 10001] {
let input = UploadPartInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.upload_id("upload-id".to_string())
.part_number(part_number)
.build()
.unwrap();
let req = build_request(input, Method::PUT);
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
}
}
#[tokio::test]
#[serial_test::serial]
async fn execute_upload_part_rejects_unknown_length_before_admission_or_body_polling() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations, MakeBucketOptions};
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
let bucket = format!("upload-part-length-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket");
let concurrency_manager = Arc::new(ConcurrencyManager::with_large_put_admission_for_test(
true,
1,
rustfs_config::DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES,
Duration::ZERO,
));
let held = concurrency_manager
.admit_multipart_part(1024)
.await
.expect("hold the only permit");
let usecase = DefaultMultipartUsecase::with_context_and_concurrency_manager(Some(context), concurrency_manager);
for capped in [false, true] {
let mut options = ObjectOptions::default();
if capped {
insert_str(&mut options.user_defined, SUFFIX_MAX_TOTAL_OBJECT_SIZE, "1024".to_owned());
}
let upload = store
.new_multipart_upload(&bucket, "object", &options)
.await
.expect("upload session");
for content_length in [None, Some(-1)] {
for declared_chunk_encoding in [false, true] {
let (body, polls) = crate::app::object::PollCountingBody::streaming_blob();
let input = UploadPartInput::builder()
.bucket(bucket.clone())
.key("object".to_owned())
.upload_id(upload.upload_id.clone())
.part_number(1)
.content_length(content_length)
.body(Some(body))
.build()
.expect("part request");
let mut request = build_request(input, Method::PUT);
request
.headers
.insert("x-amz-decoded-content-length", HeaderValue::from_static("1024"));
if declared_chunk_encoding {
request
.headers
.insert("content-encoding", HeaderValue::from_static("aws-chunked"));
}
let error = usecase
.execute_upload_part(request)
.await
.expect_err("unknown or negative logical size");
assert_eq!(
error.code(),
&if capped || content_length.is_some() {
S3ErrorCode::UnexpectedContent
} else {
S3ErrorCode::MissingContentLength
}
);
assert_eq!(polls.load(std::sync::atomic::Ordering::Relaxed), 0);
}
}
let parts = store
.list_object_parts(&bucket, "object", &upload.upload_id, None, 1000, &ObjectOptions::default())
.await
.expect("list rejected session");
assert!(parts.parts.is_empty());
}
drop(held);
}
#[tokio::test]
#[serial_test::serial]
async fn execute_upload_part_progress_timeout_releases_admission_and_leaves_no_part() {
use crate::app::object::request_body::{BodyReadControl, ObservedBody};
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
use http_body::Frame;
use http_body_util::StreamBody;
use tokio_stream::wrappers::UnboundedReceiverStream;
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
let ambient = crate::app::runtime_sources::current_app_context().expect("test app context");
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
let bucket = format!("part-progress-{}", Uuid::new_v4().simple());
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
let manager = Arc::new(ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO));
let usecase = DefaultMultipartUsecase::with_context_and_concurrency_manager(Some(context), Arc::clone(&manager));
let upload = store
.new_multipart_upload(&bucket, "object", &ObjectOptions::default())
.await
.unwrap();
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<Result<Frame<Bytes>, std::io::Error>>();
let control = BodyReadControl::default();
let body = StreamingBlob::from(crate::app::storage_api::s3::Body::http_body_unsync(ObservedBody::new(
StreamBody::new(UnboundedReceiverStream::new(receiver)),
control.clone(),
)));
let input = UploadPartInput::builder()
.bucket(bucket.clone())
.key("object".to_owned())
.upload_id(upload.upload_id.clone())
.part_number(1)
.content_length(Some(65536))
.body(Some(body))
.build()
.unwrap();
let mut request = build_request(input, Method::PUT);
request.extensions.insert(control.clone());
let owner = tokio::spawn(async move { usecase.execute_upload_part(request).await });
control.wait_until_pending_for_test().await;
assert!(matches!(
manager.admit_put_object(65536).await.unwrap(),
ForegroundWriteAdmission::Rejected
));
control.advance_wait_for_test(Duration::from_secs(60));
sender.send(Ok(Frame::data(Bytes::from_static(b"x")))).unwrap();
assert_eq!(owner.await.unwrap().unwrap_err().code(), &S3ErrorCode::RequestTimeout);
assert!(matches!(
manager.admit_put_object(65536).await.unwrap(),
ForegroundWriteAdmission::Admitted(_)
));
let parts = store
.list_object_parts(&bucket, "object", &upload.upload_id, None, 1000, &ObjectOptions::default())
.await
.unwrap();
assert!(parts.parts.is_empty());
}
#[tokio::test]
#[serial_test::serial]
async fn execute_upload_part_rejects_when_foreground_write_admission_is_full() {
use crate::app::storage_api::test::contract::bucket::{BucketOperations, MakeBucketOptions};
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
let bucket = format!("upload-part-admission-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("create upload part admission test bucket");
let upload = store
.new_multipart_upload(&bucket, "object", &ObjectOptions::default())
.await
.expect("create multipart upload");
let concurrency_manager = Arc::new(ConcurrencyManager::with_large_put_admission_for_test(
true,
1,
rustfs_config::DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES,
Duration::ZERO,
));
let held = concurrency_manager
.admit_multipart_part(1024)
.await
.expect("first multipart part should acquire the only permit");
let usecase =
DefaultMultipartUsecase::with_context_and_concurrency_manager(Some(context), Arc::clone(&concurrency_manager));
let body = StreamingBlob::wrap(futures::stream::pending::<Result<Bytes, std::io::Error>>());
let input = UploadPartInput::builder()
.bucket(bucket)
.key("object".to_string())
.upload_id(upload.upload_id)
.part_number(1)
.content_length(Some(1024))
.body(Some(body))
.build()
.expect("upload part input should build");
let err = usecase
.execute_upload_part(build_request(input, Method::PUT))
.await
.expect_err("full foreground write admission should reject before body ingest");
assert_eq!(err.code(), &S3ErrorCode::SlowDown);
drop(held);
}
mod body_read_tests;
}
@@ -0,0 +1,556 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! UploadPartCopy.
use super::*;
fn normalize_copy_source_version_id(version_id: Option<&str>) -> S3Result<Option<String>> {
match version_id {
Some(v) => {
let trimmed = v.trim();
if trimmed.eq_ignore_ascii_case("null") {
Ok(Some(Uuid::nil().to_string()))
} else if Uuid::parse_str(trimmed).is_ok() {
Ok(Some(trimmed.to_string()))
} else {
Err(S3Error::with_message(
S3ErrorCode::InvalidArgument,
"Invalid version id specified in copy source",
))
}
}
None => Ok(None),
}
}
fn format_copy_source_version_id(version_id: Option<Uuid>, src_versioned: bool, src_version_suspended: bool) -> Option<String> {
if src_versioned || src_version_suspended {
Some(match version_id {
Some(vid) if vid != Uuid::nil() => vid.to_string(),
_ => "null".to_string(),
})
} else {
None
}
}
/// Returns InvalidRange error if CopySourceRange end exceeds the source object size.
/// Used by execute_upload_part_copy to reject out-of-bounds ranges per S3 spec.
fn validate_copy_source_range_not_exceeds(range_spec: &HTTPRangeSpec, object_size: i64) -> S3Result<()> {
if range_spec.end >= object_size {
return Err(S3Error::with_message(
S3ErrorCode::InvalidRange,
"The requested range is not satisfiable".to_string(),
));
}
Ok(())
}
impl DefaultMultipartUsecase {
#[instrument(level = "debug", skip(self, req))]
#[hotpath::measure(impl_type = "MultipartUsecase")]
pub async fn execute_upload_part_copy(
&self,
req: S3Request<UploadPartCopyInput>,
) -> S3Result<S3Response<UploadPartCopyOutput>> {
reject_presigned_multipart_max_total_object_size_for_other_operation(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedSigV4Request>().is_some(),
)?;
reject_presigned_put_max_content_length_for_other_operation(
&req.headers,
req.uri.query(),
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
)?;
// Captured before `req.input` is destructured below.
let copy_principal = SseKmsPrincipal::from_request(&req);
let source_bucket = match &req.input.copy_source {
CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)),
CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)),
CopySource::Bucket { bucket, .. } => bucket.to_string(),
};
let mut source_generation_opts = ObjectOptions::default();
apply_copy_source_bucket_generation_guard(&req, &source_bucket, &mut source_generation_opts)?;
let expected_source_incarnation_id = source_generation_opts.expected_bucket_incarnation_id;
let mut destination_generation_opts = ObjectOptions::default();
apply_bucket_generation_guard(&req, &req.input.bucket, &mut destination_generation_opts)?;
let expected_destination_incarnation_id = destination_generation_opts.expected_bucket_incarnation_id;
let UploadPartCopyInput {
bucket,
key,
copy_source,
copy_source_range,
part_number,
upload_id,
copy_source_if_match,
copy_source_if_none_match,
sse_customer_algorithm,
sse_customer_key,
sse_customer_key_md5,
copy_source_sse_customer_algorithm,
copy_source_sse_customer_key,
copy_source_sse_customer_key_md5,
..
} = req.input;
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
let (src_bucket, src_key, raw_src_version_id) = match copy_source {
CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)),
CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)),
CopySource::Bucket {
bucket: ref src_bucket,
key: ref src_key,
version_id,
} => (src_bucket.to_string(), src_key.to_string(), version_id.map(|v| v.to_string())),
};
let src_version_id = normalize_copy_source_version_id(raw_src_version_id.as_deref())?;
let rs = if let Some(range_str) = copy_source_range {
Some(parse_copy_source_range(&range_str)?)
} else {
None
};
let part_id = parse_upload_part_number(part_number)?;
validate_table_catalog_object_mutation(&bucket, &key).await?;
let Some(store) = self.object_store() else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let (
source_bucket_lifecycle_guard,
current_source_incarnation_id,
destination_bucket_lifecycle_guard_storage,
current_destination_incarnation_id,
) = store
.acquire_copy_bucket_incarnation_fences(
&src_bucket,
expected_source_incarnation_id,
&bucket,
expected_destination_incarnation_id,
)
.await
.map_err(ApiError::from)?;
let destination_bucket_lifecycle_guard = destination_bucket_lifecycle_guard_storage
.as_ref()
.unwrap_or(&source_bucket_lifecycle_guard);
if source_bucket_lifecycle_guard.is_lock_lost() || destination_bucket_lifecycle_guard.is_lock_lost() {
return Err(ApiError::from(StorageError::NamespaceLockQuorumUnavailable {
mode: "copy_bucket_generation",
bucket: bucket.clone(),
object: key.clone(),
required: 1,
achieved: 0,
})
.into());
}
let mut dst_opts = ObjectOptions {
expected_bucket_incarnation_id: Some(current_destination_incarnation_id),
..Default::default()
};
if src_bucket != bucket {
source_bucket_lifecycle_guard.attach_to_object_options(&mut dst_opts);
}
destination_bucket_lifecycle_guard.attach_to_object_options(&mut dst_opts);
let mp_info = store
.get_multipart_info(&bucket, &key, &upload_id, &dst_opts)
.await
.map_err(multipart_upload_api_error)?;
authorize_sse_kms_multipart_write(copy_principal.as_ref(), &mp_info.user_defined).await?;
let destination_size_limit = multipart_max_total_object_size(&mp_info.user_defined)?;
EncryptionRequest {
bucket: &bucket,
key: &key,
server_side_encryption: None,
ssekms_key_id: None,
ssekms_context: None,
sse_customer_algorithm: sse_customer_algorithm.clone(),
sse_customer_key: sse_customer_key.clone(),
sse_customer_key_md5: sse_customer_key_md5.clone(),
content_size: 0,
principal: None,
}
.validate_multipart_ssec(&mp_info.user_defined)?;
let src_versioned = BucketVersioningSys::prefix_enabled(&src_bucket, &src_key).await;
let src_version_suspended = BucketVersioningSys::prefix_suspended(&src_bucket, &src_key).await;
let mut src_opts = copy_src_opts(&src_bucket, &src_key, &req.headers).map_err(ApiError::from)?;
src_opts.version_id = src_version_id.clone();
src_opts.versioned = src_versioned;
src_opts.version_suspended = src_version_suspended;
let h = build_ssec_read_headers(
copy_source_sse_customer_algorithm.as_ref(),
copy_source_sse_customer_key.as_ref(),
copy_source_sse_customer_key_md5.as_ref(),
);
let get_opts = ObjectOptions {
version_id: src_opts.version_id.clone(),
versioned: src_opts.versioned,
version_suspended: src_opts.version_suspended,
expected_bucket_incarnation_id: Some(current_source_incarnation_id),
..Default::default()
};
if source_bucket_lifecycle_guard.is_lock_lost() {
return Err(ApiError::from(StorageError::NamespaceLockQuorumUnavailable {
mode: "copy_source_bucket_generation",
bucket: src_bucket.clone(),
object: src_key.clone(),
required: 1,
achieved: 0,
})
.into());
}
let prepared_source = store
.prepare_get_object_reader(&src_bucket, &src_key, rs.clone(), h, &get_opts)
.await
.map_err(map_get_object_reader_error)?;
let src_info = prepared_source.object_info();
if src_info.delete_marker {
let error = if src_opts.version_id.is_none() {
StorageError::ObjectNotFound(src_bucket.clone(), src_key.clone())
} else {
StorageError::MethodNotAllowed
};
return Err(map_get_object_reader_error(error).into());
}
// Same shape as CopyObject: the part copy reads the source plaintext, and the source
// read resolves its material inside the object layer, which carries no request identity.
authorize_sse_kms_object_read(copy_principal.as_ref(), &src_info.user_defined).await?;
let resolved_src_version_id = format_copy_source_version_id(src_info.version_id, src_versioned, src_version_suspended);
if let Some(if_match) = copy_source_if_match {
if let Some(ref etag) = src_info.etag {
if let Some(strong_etag) = if_match.into_etag() {
if ETag::Strong(etag.clone()) != strong_etag {
return Err(s3_error!(PreconditionFailed));
}
} else {
return Err(s3_error!(PreconditionFailed));
}
} else {
return Err(s3_error!(PreconditionFailed));
}
}
if let Some(if_none_match) = copy_source_if_none_match
&& let Some(ref etag) = src_info.etag
&& let Some(strong_etag) = if_none_match.into_etag()
&& ETag::Strong(etag.clone()) == strong_etag
{
return Err(s3_error!(PreconditionFailed));
}
let source_actual_size = src_info.get_actual_size().ok().filter(|size| *size >= 0);
let source_logical_size = match source_actual_size {
Some(size) => size,
None if destination_size_limit.is_some() => {
return Err(S3Error::new(S3ErrorCode::UnexpectedContent));
}
None => src_info.size,
};
let (_start_offset, length) = if let Some(ref range_spec) = rs {
// Copy-source ranges are expressed over the logical plaintext object.
// Encrypted (and compressed) objects have a larger or smaller physical
// representation, so validating against `size` rejects valid later parts.
validate_copy_source_range_not_exceeds(range_spec, source_logical_size)?;
range_spec
.get_offset_length(source_logical_size)
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRange, e.to_string()))?
} else {
(0, source_logical_size)
};
// Resolve the range under the prepared source's read lock, then try the
// shared budget without waiting while holding namespace/lifecycle locks.
// Keep both the budget and source cancellation alive through the write.
let admission_length = source_actual_size.map_or(-1, |_| length);
let _copy_admission = match self.concurrency_manager().try_admit_multipart_part_copy(admission_length) {
ForegroundWriteAdmission::Disabled => None,
ForegroundWriteAdmission::Admitted(permit) => Some(permit),
ForegroundWriteAdmission::Rejected => {
return Err(S3Error::with_message(
S3ErrorCode::SlowDown,
"foreground write concurrency limit reached, please reduce your request rate",
));
}
};
let (src_reader, _source_cancellation) = prepared_source
.into_reader_for_copy()
.await
.map_err(map_get_object_reader_error)?;
let src_stream = src_reader.stream;
let is_disk_compressed =
rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
let actual_size = length;
let mut write_plan = WritePlan::new();
let mut reader = if is_disk_compressed {
let algorithm = CompressionAlgorithm::default();
let hrd = HashReader::from_stream(src_stream, length, actual_size, None, None, false).map_err(ApiError::from)?;
write_plan = write_plan.with_compression(algorithm);
hrd
} else {
HashReader::from_stream(src_stream, length, actual_size, None, None, false).map_err(ApiError::from)?
};
let server_side_encryption = mp_info
.user_defined
.get("x-amz-server-side-encryption")
.map(|s| {
ServerSideEncryption::from_str(s)
.map_err(|e| ApiError::from(StorageError::other(format!("Invalid server-side encryption: {e}"))))
})
.transpose()?;
let has_ssec = mp_info
.user_defined
.contains_key("x-amz-server-side-encryption-customer-algorithm");
let ssekms_key_id = match server_side_encryption.as_ref() {
Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => mp_info
.user_defined
.get("x-amz-server-side-encryption-aws-kms-key-id")
.map(|s| s.to_string()),
_ => None,
};
let (requested_sse, requested_kms_key_id, dst_user_defined) = if has_ssec {
let ssec_material = sse_decryption(DecryptionRequest {
bucket: &bucket,
key: &key,
metadata: &mp_info.user_defined,
sse_customer_key: sse_customer_key.as_ref(),
sse_customer_key_md5: sse_customer_key_md5.as_ref(),
principal: None,
})
.await?
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
let ssec_write = match ssec_material.key_kind {
EncryptionKeyKind::Object => WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32),
EncryptionKeyKind::Direct => {
WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id)
}
};
write_plan = write_plan.with_encryption(ssec_write);
(
Some(ssec_material.server_side_encryption),
ssec_material.kms_key_id,
mp_info.user_defined.clone(),
)
} else if let Some(server_side_encryption) = server_side_encryption {
// Reuse the destination envelope after authorizing this caller against
// its pinned key above. The source read is separately authorized.
let managed_material = sse_decryption(DecryptionRequest {
bucket: &bucket,
key: &key,
metadata: &mp_info.user_defined,
sse_customer_key: None,
sse_customer_key_md5: None,
principal: None,
})
.await?
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
let managed_write = match managed_material.key_kind {
EncryptionKeyKind::Object => WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32),
EncryptionKeyKind::Direct => {
WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_id)
}
};
write_plan = write_plan.with_encryption(managed_write);
(Some(server_side_encryption), ssekms_key_id, mp_info.user_defined.clone())
} else {
(None, None, mp_info.user_defined.clone())
};
reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
if let Some(checksum_algorithm) = mp_info
.user_defined
.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)
.filter(|checksum_algorithm| !checksum_algorithm.is_empty())
{
let checksum_type = rustfs_rio::ChecksumType::from_string_with_obj_type(
checksum_algorithm,
mp_info
.user_defined
.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE)
.map(String::as_str)
.unwrap_or_default(),
);
if !checksum_type.is_set() {
return Err(ApiError::from(StorageError::other(format!(
"Invalid multipart checksum type: {checksum_algorithm}"
)))
.into());
}
reader.add_calculated_checksum(checksum_type).map_err(ApiError::from)?;
}
let mut reader = PutObjReader::new(reader);
dst_opts.user_defined = dst_user_defined;
let part_info = store
.put_object_part(&bucket, &key, &upload_id, part_id, &mut reader, &dst_opts)
.await
.map_err(ApiError::from)?;
let copy_checksums = reader.as_hash_reader().content_crc();
let checksum_value = |checksum_type: rustfs_rio::ChecksumType| copy_checksums.get(&checksum_type.to_string()).cloned();
let copy_part_result = CopyPartResult {
checksum_crc32: checksum_value(rustfs_rio::ChecksumType::CRC32),
checksum_crc32c: checksum_value(rustfs_rio::ChecksumType::CRC32C),
checksum_sha1: checksum_value(rustfs_rio::ChecksumType::SHA1),
checksum_sha256: checksum_value(rustfs_rio::ChecksumType::SHA256),
checksum_crc64nvme: checksum_value(rustfs_rio::ChecksumType::CRC64_NVME),
e_tag: part_info.etag.map(|etag| to_s3s_xml_etag(&etag)),
last_modified: part_info.last_mod.map(Timestamp::from),
..Default::default()
};
let output = UploadPartCopyOutput {
copy_part_result: Some(copy_part_result),
copy_source_version_id: resolved_src_version_id,
server_side_encryption: requested_sse,
ssekms_key_id: requested_kms_key_id,
sse_customer_algorithm,
sse_customer_key_md5,
..Default::default()
};
Ok(S3Response::new(output))
}
}
#[cfg(test)]
mod tests {
use super::super::test_support::*;
use super::*;
#[tokio::test]
async fn execute_upload_part_copy_returns_internal_error_when_store_uninitialized() {
let input = UploadPartCopyInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.copy_source(CopySource::Bucket {
bucket: "src-bucket".into(),
key: "src-object".into(),
version_id: None,
})
.part_number(1)
.upload_id("upload-id".to_string())
.build()
.unwrap();
let req = build_request(input, Method::PUT);
let err = Box::pin(make_usecase().execute_upload_part_copy(req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
}
#[tokio::test]
async fn execute_upload_part_copy_rejects_invalid_part_number_before_store_lookup() {
for part_number in [-1, 0, 10001] {
let input = UploadPartCopyInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.copy_source(CopySource::Bucket {
bucket: "src-bucket".into(),
key: "src-object".into(),
version_id: None,
})
.part_number(part_number)
.upload_id("upload-id".to_string())
.build()
.unwrap();
let req = build_request(input, Method::PUT);
let err = Box::pin(make_usecase().execute_upload_part_copy(req)).await.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
}
}
#[test]
fn test_validate_copy_source_range_not_exceeds_returns_invalid_range_when_range_exceeds() {
use super::validate_copy_source_range_not_exceeds;
let range_exceeds = HTTPRangeSpec {
is_suffix_length: false,
start: 0,
end: 21,
};
let err = validate_copy_source_range_not_exceeds(&range_exceeds, 5).unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InvalidRange);
assert!(err.to_string().contains("not satisfiable"));
}
#[test]
fn test_validate_copy_source_range_not_exceeds_ok_when_range_valid() {
use super::validate_copy_source_range_not_exceeds;
let range_valid = HTTPRangeSpec {
is_suffix_length: false,
start: 0,
end: 4,
};
assert!(validate_copy_source_range_not_exceeds(&range_valid, 5).is_ok());
}
#[test]
fn test_validate_copy_source_range_not_exceeds_ok_for_suffix_range() {
use super::validate_copy_source_range_not_exceeds;
let range_suffix = HTTPRangeSpec {
is_suffix_length: true,
start: -5,
end: -1,
};
assert!(validate_copy_source_range_not_exceeds(&range_suffix, 5).is_ok());
}
#[test]
fn upload_part_copy_normalizes_and_formats_source_version_id() {
assert_eq!(normalize_copy_source_version_id(None).expect("missing version id"), None);
assert_eq!(
normalize_copy_source_version_id(Some(" null ")).expect("null version id"),
Some(Uuid::nil().to_string())
);
let valid = Uuid::new_v4();
assert_eq!(
normalize_copy_source_version_id(Some(&format!(" {valid} "))).expect("valid uuid"),
Some(valid.to_string())
);
let err = normalize_copy_source_version_id(Some("not-a-uuid")).expect_err("invalid uuid should fail");
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(format_copy_source_version_id(Some(Uuid::nil()), false, false), None);
assert_eq!(format_copy_source_version_id(None, true, false), Some("null".to_string()));
assert_eq!(format_copy_source_version_id(Some(Uuid::nil()), false, true), Some("null".to_string()));
assert_eq!(format_copy_source_version_id(Some(valid), true, false), Some(valid.to_string()));
}
mod copy_admission_tests;
}
File diff suppressed because it is too large Load Diff
+13 -34
View File
@@ -53,16 +53,10 @@ pub(crate) mod s3 {
PersistedReplicationFilter, PersistedReplicationRule, PersistedReplicationTag,
};
#[cfg(test)]
pub(crate) use s3s::access::S3Access;
#[cfg(test)]
pub(crate) use s3s::auth::{Credentials, SimpleAuth};
#[cfg(test)]
pub(crate) use s3s::config::{S3Config, StaticConfigProvider};
#[cfg(test)]
pub(crate) use s3s::dto::{
BucketLocationConstraint, CreateBucketConfiguration, CreateBucketInput, CreateMultipartUploadInput, DeleteObjectInput,
ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, PutObjectInput, Tag, Tagging,
};
pub(crate) use s3s::dto;
#[cfg(test)]
pub(crate) use s3s::dto::{GetObjectInput, HeadObjectInput, ListObjectsV2Input, ListObjectsV2Output};
#[cfg(test)]
@@ -72,6 +66,8 @@ pub(crate) mod s3 {
#[cfg(test)]
pub(crate) use s3s::stream::upload_stream::{UploadStream, UploadStreamError};
#[cfg(test)]
pub(crate) use s3s::xml;
#[cfg(test)]
pub(crate) use s3s::xml::{Serialize as XmlSerialize, Serializer as XmlSerializer};
#[cfg(test)]
pub(crate) use s3s::{Body, BodySizeLimitExceeded, S3, S3Response};
@@ -317,11 +313,10 @@ pub(crate) mod access {
#[cfg(test)]
pub(crate) use crate::storage::storage_api::access_consumer::ReqInfo;
pub(crate) use crate::storage::storage_api::access_consumer::{
ListBucketsAuthzRequest, PostObjectRequestMarker, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess,
TableDataPlaneListCursorPosition, TableDataPlanePublicationGuards, apply_bucket_generation_guard,
apply_copy_source_bucket_generation_guard, authorize_copy_object_tags, authorize_request,
bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header,
load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
PostObjectRequestMarker, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, TableDataPlaneListCursorPosition,
TableDataPlanePublicationGuards, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard,
authorize_copy_object_tags, authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action,
has_bypass_governance_header, load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller,
replication_proxy_allowed, replication_request_authorized, req_info_mut, req_info_ref,
retained_table_data_plane_publication_guards, set_requested_object_lock_retain_until,
@@ -1673,9 +1668,9 @@ pub(crate) mod timeout_wrapper {
pub(crate) mod s3_api {
pub(crate) mod bucket {
pub(crate) use crate::storage::storage_api::s3_api_consumer::bucket::{
ListObjectVersionsParams, ListObjectsV2Params, build_list_object_versions_output, build_list_objects_output,
build_list_objects_v2_output, encode_list_objects_v2_continuation_token, parse_list_object_versions_params,
parse_list_objects_v2_params,
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
parse_list_object_versions_params, parse_list_objects_v2_params,
};
pub(crate) use crate::storage::storage_api::s3_api_consumer::common::rustfs_owner;
}
@@ -1711,7 +1706,7 @@ pub(crate) mod bucket_usecase {
pub(crate) mod contract {
pub(crate) mod bucket {
pub(crate) use super::super::super::storage_contracts::{
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
};
}
@@ -1823,30 +1818,14 @@ pub(crate) mod select_object {
};
}
/// The storage items the owned/legacy authorization parity tests drive.
#[cfg(test)]
pub(crate) mod owned_parity {
pub(crate) use crate::storage::storage_api::owned_parity_consumer::{
DenialAuditor, DenialRateLimiter, DenialScope, FS, ObjectTagConditions, PostObjectPolicyHeaders, PostObjectRequestMarker,
ReqInfo, RequestContext, RequestedCopySource, RequestedObjectLegalHold, RequestedObjectLockRetainUntil,
RequestedObjectLockRetention, RequestedObjectTags, SUPPRESSED_DENIALS_TAG, TableDataPlanePublicationGuards,
authorization_denial_entry, authorization_denial_entry_for_view, authorize_request, parity_test_support,
};
}
pub(crate) mod context {
#[cfg(test)]
pub(crate) use super::EndpointServerPools;
pub(crate) use super::bucket;
pub(crate) use super::runtime;
pub(crate) use crate::storage::storage_api::access_consumer::{
AuditRequestView, AuthorizationRequest, DenialAuditor, DenialScope, authorize_request_core,
};
pub(crate) use crate::storage::storage_api::access_consumer::{
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, MultipartStagingOperation,
ObjectTagConditions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectPolicyHeaders,
PostObjectRequestMarker, ReqInfo, RequestedCopySource, RequestedObjectLegalHold, RequestedObjectLockRetainUntil,
RequestedObjectLockRetention, RequestedObjectTags, StagedMultipartPartAuthorization, TableDataPlaneListAccess,
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, ObjectTagConditions,
OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectRequestMarker, ReqInfo, TableDataPlaneListAccess,
TableDataPlanePublicationGuards,
};
pub(crate) use crate::storage::storage_api::request_context_consumer::RequestContext;
+1 -1
View File
@@ -78,7 +78,7 @@ impl FromStr for S3Stack {
impl S3Stack {
/// Unset selects the default; a set but invalid value, including an empty
/// one, is a startup error.
pub(crate) fn from_env() -> std::io::Result<Self> {
fn from_env() -> std::io::Result<Self> {
match std::env::var_os(ENV_RUSTFS_S3_STACK) {
None => Ok(Self::default()),
Some(value) => value
+287
View File
@@ -0,0 +1,287 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Test-only source reader for text guards that must survive a mechanical file split.
//!
//! A text guard asserts that some literal appears in production source. When it reads one file
//! with `include_str!`, moving the guarded function into another file either breaks the guard or,
//! worse, makes it stop seeing the code it guards. A guard built on [`SourceSet`] instead names
//! the file it reads today plus the sibling directory a split moves that file's code into, and
//! reads every `.rs` file of both at test time.
//!
//! The named file is required: a missing file panics with its path, so a guard never passes on
//! absent input. The directory is optional while the split has not happened; once it exists,
//! every `.rs` file under it (recursively) is read in byte order of its manifest-relative path,
//! after the named file.
use std::path::{Path, PathBuf};
/// One source file of a [`SourceSet`].
pub(crate) struct SourceFile {
/// Path relative to the `rustfs` package root, with `/` separators.
pub(crate) path: String,
pub(crate) text: String,
}
/// The ordered files a text guard reads: one required file, then the split directory.
pub(crate) struct SourceSet {
files: Vec<SourceFile>,
}
impl SourceSet {
/// Reads `file` and every `.rs` file under `split_dir`, both relative to the `rustfs`
/// package root.
///
/// # Panics
///
/// Panics, naming the path, when `file` is missing or unreadable, when `split_dir` exists but
/// is not a directory, or when a file under `split_dir` cannot be read.
pub(crate) fn read(file: &str, split_dir: &str) -> Self {
Self::read_under(Path::new(env!("CARGO_MANIFEST_DIR")), file, split_dir)
}
fn read_under(root: &Path, file: &str, split_dir: &str) -> Self {
let required = root.join(file);
let text = std::fs::read_to_string(&required)
.unwrap_or_else(|err| panic!("guard source {} must exist and be readable: {err}", required.display()));
let mut files = vec![SourceFile {
path: file.to_owned(),
text,
}];
let dir = root.join(split_dir);
if dir.exists() {
assert!(dir.is_dir(), "guard split path {} must be a directory", dir.display());
let mut paths = Vec::new();
collect_rust_files(&dir, &mut paths);
let mut split_files: Vec<SourceFile> = paths
.into_iter()
.map(|path| {
let text = std::fs::read_to_string(&path)
.unwrap_or_else(|err| panic!("guard source {} must be readable: {err}", path.display()));
let relative = path
.strip_prefix(root)
.unwrap_or_else(|_| panic!("guard source {} must be under {}", path.display(), root.display()));
let path = relative
.components()
.map(|component| component.as_os_str().to_string_lossy())
.collect::<Vec<_>>()
.join("/");
SourceFile { path, text }
})
.collect();
split_files.sort_by(|left, right| left.path.cmp(&right.path));
files.extend(split_files);
}
Self { files }
}
/// Every file of the set, the required file first.
pub(crate) fn files(&self) -> &[SourceFile] {
&self.files
}
/// Whether any one file of the set contains `needle`.
pub(crate) fn contains(&self, needle: &str) -> bool {
self.files.iter().any(|file| file.text.contains(needle))
}
/// Non-overlapping occurrences of `needle`, summed over the files; a match never spans two
/// files.
pub(crate) fn count(&self, needle: &str) -> usize {
self.files.iter().map(|file| file.text.matches(needle).count()).sum()
}
/// The text after `marker` in the one file that contains it.
///
/// # Panics
///
/// Panics unless `marker` occurs exactly once across the whole set.
pub(crate) fn after_unique_marker(&self, marker: &str) -> &str {
let occurrences = self.count(marker);
assert_eq!(occurrences, 1, "marker {marker:?} must occur exactly once across {}", self.describe());
self.files
.iter()
.find_map(|file| file.text.split_once(marker).map(|(_, rest)| rest))
.unwrap_or_else(|| unreachable!("a counted marker is found"))
}
/// Manifest-relative paths of the set, for panic messages.
pub(crate) fn describe(&self) -> String {
self.files
.iter()
.map(|file| file.path.as_str())
.collect::<Vec<_>>()
.join(", ")
}
}
fn collect_rust_files(dir: &Path, out: &mut Vec<PathBuf>) {
let entries =
std::fs::read_dir(dir).unwrap_or_else(|err| panic!("guard split directory {} must be readable: {err}", dir.display()));
for entry in entries {
let entry = entry.unwrap_or_else(|err| panic!("guard split directory {} must be listable: {err}", dir.display()));
let path = entry.path();
if path.is_dir() {
collect_rust_files(&path, out);
} else if path.extension().is_some_and(|extension| extension == "rs") {
out.push(path);
}
}
}
#[cfg(test)]
mod tests {
use super::SourceSet;
use std::fs;
use std::path::Path;
fn write(root: &Path, relative: &str, text: &str) {
let path = root.join(relative);
fs::create_dir_all(path.parent().expect("fixture path has a parent")).expect("create fixture dir");
fs::write(path, text).expect("write fixture");
}
fn panic_message(run: impl FnOnce() + std::panic::UnwindSafe) -> String {
let payload = std::panic::catch_unwind(run).expect_err("the guard input must be rejected");
payload
.downcast_ref::<String>()
.cloned()
.or_else(|| payload.downcast_ref::<&str>().map(|message| (*message).to_owned()))
.unwrap_or_default()
}
#[test]
fn absent_split_directory_reads_only_the_required_file() {
let root = tempfile::tempdir().expect("tempdir");
write(root.path(), "src/app/usecase.rs", "guarded();\n");
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
assert_eq!(set.describe(), "src/app/usecase.rs");
assert!(set.contains("guarded();"));
assert_eq!(set.count("guarded();"), 1);
}
#[test]
fn split_directory_files_follow_the_required_file_in_path_order() {
let root = tempfile::tempdir().expect("tempdir");
write(root.path(), "src/app/usecase.rs", "guarded();\n");
write(root.path(), "src/app/usecase/zeta.rs", "guarded();\n");
write(root.path(), "src/app/usecase/alpha.rs", "guarded();\nguarded();\n");
write(root.path(), "src/app/usecase/tests/nested.rs", "guarded();\n");
write(root.path(), "src/app/usecase/notes.txt", "guarded();\n");
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
assert_eq!(
set.describe(),
"src/app/usecase.rs, src/app/usecase/alpha.rs, src/app/usecase/tests/nested.rs, src/app/usecase/zeta.rs"
);
assert_eq!(set.count("guarded();"), 5, "non-.rs files are not source");
}
#[test]
fn missing_required_file_fails_even_when_the_split_directory_exists() {
let root = tempfile::tempdir().expect("tempdir");
write(root.path(), "src/app/usecase/moved.rs", "guarded();\n");
let root_path = root.path().to_path_buf();
let message = panic_message(move || {
SourceSet::read_under(&root_path, "src/app/usecase.rs", "src/app/usecase");
});
assert!(message.contains("src/app/usecase.rs"), "panic must name the missing file: {message}");
}
#[test]
fn split_path_that_is_a_file_fails() {
let root = tempfile::tempdir().expect("tempdir");
write(root.path(), "src/app/usecase.rs", "guarded();\n");
write(root.path(), "src/app/usecase", "not a directory\n");
let root_path = root.path().to_path_buf();
let message = panic_message(move || {
SourceSet::read_under(&root_path, "src/app/usecase.rs", "src/app/usecase");
});
assert!(message.contains("must be a directory"), "{message}");
}
#[test]
fn count_does_not_join_a_needle_across_two_files() {
let root = tempfile::tempdir().expect("tempdir");
write(root.path(), "src/app/usecase.rs", "guard");
write(root.path(), "src/app/usecase/tail.rs", "ed();\n");
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
assert_eq!(set.count("guarded();"), 0);
assert!(!set.contains("guarded();"));
}
#[test]
fn unique_marker_absent_from_the_set_fails() {
let root = tempfile::tempdir().expect("tempdir");
write(root.path(), "src/app/usecase.rs", "fn other() {}\n");
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
let message = panic_message(move || {
set.after_unique_marker("fn guarded");
});
assert!(message.contains("exactly once"), "{message}");
}
#[test]
fn unique_marker_in_two_files_fails() {
let root = tempfile::tempdir().expect("tempdir");
write(root.path(), "src/app/usecase.rs", "fn guarded() {}\n");
write(root.path(), "src/app/usecase/copy.rs", "fn guarded() {}\n");
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
let message = panic_message(move || {
set.after_unique_marker("fn guarded");
});
assert!(message.contains("src/app/usecase/copy.rs"), "panic must name the set: {message}");
}
#[test]
fn unique_marker_twice_in_one_file_fails() {
let root = tempfile::tempdir().expect("tempdir");
write(root.path(), "src/app/usecase.rs", "fn guarded() {}\nfn guarded() {}\n");
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
let message = panic_message(move || {
set.after_unique_marker("fn guarded");
});
assert!(message.contains("exactly once"), "{message}");
}
#[test]
fn unique_marker_returns_the_rest_of_its_own_file_only() {
let root = tempfile::tempdir().expect("tempdir");
write(root.path(), "src/app/usecase.rs", "fn head() {}\n");
write(root.path(), "src/app/usecase/moved.rs", "fn guarded() { body(); }\n");
write(root.path(), "src/app/usecase/next.rs", "fn later() { other(); }\n");
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
let rest = set.after_unique_marker("fn guarded");
assert_eq!(rest, "() { body(); }\n");
}
}
+2
View File
@@ -89,6 +89,8 @@ pub mod diagnose;
pub mod embedded;
pub mod error;
pub mod foreground_observation;
#[cfg(test)]
pub(crate) mod guard_sources;
pub mod init;
pub mod inspect;
pub(crate) mod kms_deletion_gate;
@@ -1,61 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The access phase of a gateway handler: the legacy access hooks in their owned form.
//!
//! Responsible for: [`check`], what the legacy access phase runs once per request before any
//! operation hook. Per-operation hooks join it here as their operations are registered.
//! NOT responsible for: the decisions themselves. Every function here calls the function the
//! legacy access phase calls (`storage/access.rs`), so the two stacks cannot answer differently.
//! Upstream: each `impl Handler`. Downstream: the storage layer's access functions.
//!
//! The caller is resolved here on every request, and again by the gateway's `Authorizer` before
//! it: the pinned gateway hands a handler no policy snapshot, so one resolution cannot be carried
//! from the authorization stages to the handler.
use crate::app::RequestEnvelope;
use crate::runtime_sources::ServerContextSlot;
use crate::storage_api::server::http::access::{admit_operation, resolve_req_info};
use rustfs_s3_types::S3Result;
use std::sync::Arc;
/// Runs the per-request access check for `operation` and installs the caller in `envelope`.
///
/// The caller is the credential IAM holds for the access key the gateway verified, checked
/// against the presented session token; an anonymous request resolves to no caller. The signed
/// upload-size capabilities are then refused on any operation but their own, and the license is
/// checked.
///
/// # Errors
///
/// What the legacy access phase answers: the credential refusals of the key lookup,
/// `InvalidRequest` for a misplaced capability query, `AccessDenied` for a refused license, and
/// `InternalError` when IAM or the license cannot be read.
pub(crate) async fn check(
server_ctx: &Arc<ServerContextSlot>,
envelope: &mut RequestEnvelope,
operation: &'static str,
) -> S3Result<()> {
let credentials = envelope.credentials();
let access_key = credentials.map(|credentials| credentials.access_key().to_owned());
let session_token = credentials
.and_then(|credentials| credentials.session_token())
.unwrap_or_default()
.to_owned();
let request_context = envelope.request_context().cloned();
let req_info = resolve_req_info(server_ctx, access_key.as_deref(), &session_token, request_context).await?;
envelope.install_req_info(req_info);
admit_operation(envelope.headers(), envelope.uri().query(), operation)?;
Ok(())
}
@@ -1,48 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! `ListBuckets` on the gateway stack.
//!
//! Responsible for: converting one accepted request into the envelope and input the RustFS use
//! case takes, and its answer back.
//! NOT responsible for: deciding which buckets the caller sees. The legacy access hook for this
//! operation is empty and so is this one: `DefaultBucketUsecase::execute_list_buckets` decides
//! per caller (every bucket with `s3:ListAllMyBuckets`, otherwise the buckets the caller may
//! list or locate, `AccessDenied` when there is none), for both stacks.
//! Upstream: the gateway's dispatch. Downstream: `crate::app::bucket_usecase`.
use super::super::GatewayBackend;
use super::super::access;
use super::super::error::handler_error;
use crate::app::RequestEnvelope;
use crate::app::bucket_usecase::DefaultBucketUsecase;
use rustfs_gateway::{Handler, HandlerResult, Operation, Req, Resp, dto};
use rustfs_io_metrics::record_s3_op;
use rustfs_s3_ops::S3Operation;
impl Handler<dto::ListBuckets> for GatewayBackend {
async fn call(&self, request: Req<dto::ListBuckets>) -> HandlerResult<dto::ListBuckets> {
record_s3_op(S3Operation::ListBuckets);
let mut envelope = RequestEnvelope::from_gateway(request.context(), self.server_ctx()).map_err(handler_error)?;
access::check(self.server_ctx(), &mut envelope, <dto::ListBuckets as Operation>::NAME)
.await
.map_err(handler_error)?;
let usecase = DefaultBucketUsecase::with_context(self.server_ctx().app_context());
let output = usecase
.execute_list_buckets(envelope, request.into_input())
.await
.map_err(handler_error)?;
Ok(Resp::new(output))
}
}
@@ -1,17 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Bucket-level operations of the gateway stack, one file each.
mod list_buckets;
@@ -1,373 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Builds a [`RequestEnvelope`] from a request the gateway accepted, authenticated and routed.
//!
//! Responsible for: `RequestEnvelope::from_gateway`, the gateway edge's counterpart of the
//! legacy edge's `from_s3s` (`storage/s3_compat/envelope.rs`).
//! NOT responsible for: the facts the access phase attaches afterwards (`ReqInfo` and what the
//! operation hooks retain); `super::access` installs those.
//! Upstream: the gateway's handler request context. Downstream: every gateway handler.
//!
//! Every fact is copied from where the gateway or a RustFS layer in front of it put it: the
//! request line and the accepted header lines from the context, the caller from the verified
//! principal, and the payloads the connection and request layers attached from the transport
//! values the gateway carried through unchanged.
use crate::app::object::request_body::BodyReadControl;
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead};
use crate::auth::{
AuthType, VerifiedPresignedRequest, VerifiedSigV4Request, get_request_auth_type_with_query, get_session_token,
};
use crate::runtime_sources::ServerContextSlot;
use crate::shared_types::{NativeTlsTransport, RemoteAddr};
use crate::storage_api::server::http::request_context::RequestContext;
use http::{Extensions, HeaderMap, Uri};
use rustfs_gateway::{RequestContextView, TransportExtensions};
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
use rustfs_trusted_proxies::ClientInfo;
use std::sync::Arc;
impl RequestEnvelope {
/// Copies every request fact of `context` the app layer reads, for the server `server_ctx`.
///
/// A payload the request does not carry leaves its field `None`. The caller's secret key is
/// copied only when the gateway handed it to this operation; the signature markers are set
/// only for a caller the gateway verified.
///
/// # Errors
///
/// `InternalError` when the accepted request target cannot be read back as a URI, or a
/// handed-over secret key is not text. Neither is the caller's fault.
pub(crate) fn from_gateway(context: &RequestContextView, server_ctx: &Arc<ServerContextSlot>) -> S3Result<Self> {
let uri = request_uri(context)?;
let mut headers = HeaderMap::new();
for (name, value) in context.headers().iter_raw() {
headers.append(name.clone(), value.clone());
}
let credentials = match context.principal() {
Some(principal) => {
let secret_key = principal
.secret_key_from_authenticator_lookup()
.map(|secret| secret.expose_secret());
Some(request_credentials(
principal.access_key_id(),
secret_key,
get_session_token(&uri, &headers),
)?)
}
None => None,
};
let mut extensions = Extensions::new();
let transport = context.transport_extensions();
copy_payload::<Option<RemoteAddr>>(transport, &mut extensions);
copy_payload::<ClientInfo>(transport, &mut extensions);
copy_payload::<NativeTlsTransport>(transport, &mut extensions);
copy_payload::<RequestContext>(transport, &mut extensions);
copy_payload::<BodyReadControl>(transport, &mut extensions);
extensions.insert(Arc::clone(server_ctx));
// The legacy access phase marks a request whose SigV4 signature its edge verified; here
// the gateway verified it, and only a verified caller has a principal.
if credentials.is_some() {
match get_request_auth_type_with_query(&headers, uri.query()) {
AuthType::Presigned => {
extensions.insert(VerifiedPresignedRequest);
extensions.insert(VerifiedSigV4Request);
}
AuthType::Signed => {
extensions.insert(VerifiedSigV4Request);
}
_ => {}
}
}
let scope = context.verified_scope();
let head = RequestHead {
method: context.method().clone(),
uri,
headers,
credentials,
region: scope.map(|scope| scope.region().to_owned()),
service: scope.map(|scope| scope.service().to_owned()),
};
Ok(Self::from_parts(head, extensions))
}
}
/// The request target as it arrived: the raw path and query, with the scheme and authority when
/// the transport carried them (an absolute-form target, or HTTP/2).
fn request_uri(context: &RequestContextView) -> S3Result<Uri> {
let path_and_query = match context.raw_query() {
"" => context.raw_path().to_owned(),
query => format!("{}?{query}", context.raw_path()),
};
let builder = match (context.target_scheme(), context.target_authority()) {
(Some(scheme), Some(authority)) => Uri::builder().scheme(scheme).authority(authority),
_ => Uri::builder(),
};
builder
.path_and_query(path_and_query)
.build()
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "the accepted request target is not a URI"))
}
/// The caller's credentials as the app layer holds them. `secret_key` is what the gateway handed
/// over, which is nothing unless the operation opted in.
fn request_credentials(access_key: &str, secret_key: Option<&[u8]>, session_token: Option<&str>) -> S3Result<RequestCredentials> {
let session_token = session_token.map(str::to_owned);
let Some(secret_key) = secret_key else {
return Ok(RequestCredentials::without_secret_key(access_key.to_owned(), session_token));
};
let secret_key = std::str::from_utf8(secret_key)
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "the caller's secret key is not text"))?;
// The gateway refuses a credential whose secret is empty, so a handed-over secret is never
// empty. Should one arrive, the request is refused rather than served with a secret that
// every stored secret would have to be compared against.
RequestCredentials::new(access_key.to_owned(), secret_key.to_owned(), session_token)
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "the caller's secret key is empty"))
}
/// Copies the payload of type `T` a layer in front of the gateway attached, when one did.
fn copy_payload<T: Clone + Send + Sync + 'static>(transport: &TransportExtensions, extensions: &mut Extensions) {
if let Some(payload) = transport.get::<T>() {
extensions.insert(payload.clone());
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::server::gateway_handlers::error::handler_error;
use bytes::Bytes;
use http::{HeaderValue, Method, Request, StatusCode};
use http_body_util::{BodyExt, Empty};
use rustfs_gateway::sig::{AmzDate, PayloadMode, SigService, SigV4Signer, SigningCredentials, SigningRequest, SigningScope};
use rustfs_gateway::{
Credentials, Handler, HandlerResult, Limits, RegionSet, Req, Resp, ServiceBuilder, SigV4Authenticator, StaticCredentials,
WireRequest, allow_when, dto,
};
use std::net::{IpAddr, SocketAddr};
use std::sync::Mutex;
use std::time::Instant;
const PEER: ([u8; 4], u16) = ([10, 0, 0, 7], 41000);
const CLIENT: [u8; 4] = [203, 0, 113, 9];
const REQUEST_ID: &str = "envelope-request-id";
const SIGNER_ACCESS_KEY: &str = "AKIAENVELOPESIGNER";
const SIGNER_SECRET_KEY: &str = "envelope-signer-secret-key-0123456789";
/// Records the envelope the gateway edge builds for the request it is handed.
struct Probe {
server_ctx: Arc<ServerContextSlot>,
seen: Mutex<Option<RequestEnvelope>>,
}
impl Handler<dto::ListBuckets> for Probe {
async fn call(&self, request: Req<dto::ListBuckets>) -> HandlerResult<dto::ListBuckets> {
let envelope = RequestEnvelope::from_gateway(request.context(), &self.server_ctx).map_err(handler_error)?;
*self.seen.lock().expect("probe lock") = Some(envelope);
Ok(Resp::new(dto::ListBucketsOutput::default()))
}
}
/// Sends `request` through a gateway whose only handler is the probe, and returns what the
/// probe built. The test authorizer admits the one probed operation and nothing else.
async fn envelope_of(request: Request<Empty<Bytes>>, server_ctx: &Arc<ServerContextSlot>) -> RequestEnvelope {
let probe = Arc::new(Probe {
server_ctx: Arc::clone(server_ctx),
seen: Mutex::new(None),
});
let regions = RegionSet::new(["us-east-1"]).expect("a region");
let service = ServiceBuilder::new()
.rustfs_profile()
.authenticator(SigV4Authenticator::new(Arc::new(signer_credentials()), regions).rustfs_profile())
.authorizer(allow_when(|request| request.operation == "ListBuckets"))
.register::<dto::ListBuckets, _>(Arc::clone(&probe))
.build()
.expect("the probe service assembles");
let response = service.call(request).await;
let status = response.status();
if status != StatusCode::OK {
let body = response
.into_body()
.collect()
.await
.map(|body| body.to_bytes())
.unwrap_or_default();
panic!("the probe must be reached: {status} {}", String::from_utf8_lossy(&body));
}
probe
.seen
.lock()
.expect("probe lock")
.take()
.expect("the probe saw the request")
}
/// The one caller the probe service knows.
fn signer_credentials() -> StaticCredentials {
StaticCredentials::new().with(Credentials::new(SIGNER_ACCESS_KEY, SIGNER_SECRET_KEY.as_bytes()).expect("a valid key"))
}
/// A ListBuckets request header-signed now by the probe service's one caller.
fn signed_list_buckets() -> Request<Empty<Bytes>> {
let mut headers = HeaderMap::new();
headers.insert(http::header::HOST, HeaderValue::from_static("localhost"));
let probe = list_buckets_request("/").body(Bytes::new()).expect("static request");
let accepted = WireRequest::accept(probe, &Limits::default()).expect("an acceptable host");
let stamp = AmzDate::parse(&chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string()).expect("a stamp");
let scope = SigningScope::new(stamp.day(), "us-east-1", SigService::S3).expect("a scope");
let credentials = SigningCredentials::new(SIGNER_ACCESS_KEY, SIGNER_SECRET_KEY.as_bytes()).expect("credentials");
let mut signer = SigV4Signer::new(credentials, scope);
let method = Method::GET;
let signing = SigningRequest::new(
&method,
"/",
"",
&headers,
accepted.host().raw_for_signing(),
PayloadMode::Unsigned,
stamp,
);
let signed = signer.sign_headers(&signing).expect("a signable request");
let mut builder = Request::builder().method(method).uri("/");
for (name, value) in signed.headers() {
builder = builder.header(name, value);
}
builder.body(Empty::new()).expect("a request")
}
fn list_buckets_request(target: &str) -> http::request::Builder {
Request::builder().method(Method::GET).uri(target).header("host", "localhost")
}
fn request_context() -> RequestContext {
RequestContext {
request_id: REQUEST_ID.to_owned(),
x_amz_request_id: REQUEST_ID.to_owned(),
trace_id: None,
span_id: None,
start_time: Instant::now(),
}
}
#[tokio::test]
async fn from_gateway_copies_every_fact_of_the_accepted_request() {
let server_ctx = ServerContextSlot::new();
let request = list_buckets_request("/?prefix=probe")
.header("x-envelope-probe", "probe")
.extension(Some(RemoteAddr(SocketAddr::from(PEER))))
.extension(ClientInfo::direct(SocketAddr::new(IpAddr::from(CLIENT), 443)))
.extension(NativeTlsTransport(true))
.extension(request_context())
.extension(BodyReadControl::default())
.body(Empty::new())
.expect("static request");
let envelope = envelope_of(request, &server_ctx).await;
assert_eq!(envelope.method(), Method::GET);
assert_eq!(envelope.uri().path(), "/");
assert_eq!(envelope.uri().query(), Some("prefix=probe"));
assert_eq!(envelope.headers().get("x-envelope-probe"), Some(&HeaderValue::from_static("probe")));
assert_eq!(envelope.remote_addr(), Some(SocketAddr::from(PEER)));
assert_eq!(envelope.client_info().map(|info| info.real_ip), Some(IpAddr::from(CLIENT)));
assert_eq!(envelope.native_tls_transport(), Some(true));
assert_eq!(envelope.request_context().map(|context| context.request_id.as_str()), Some(REQUEST_ID));
assert!(envelope.body_read_control().is_some(), "body_read_control");
assert!(
envelope.server_context().is_some_and(|slot| Arc::ptr_eq(slot, &server_ctx)),
"server_context"
);
assert!(envelope.residual_extensions().is_empty(), "every copied payload is a named field");
}
#[tokio::test]
async fn from_gateway_leaves_absent_payloads_none_and_an_anonymous_caller_unverified() {
let server_ctx = ServerContextSlot::new();
let request = list_buckets_request("/").body(Empty::new()).expect("static request");
let envelope = envelope_of(request, &server_ctx).await;
assert!(envelope.credentials().is_none(), "credentials");
assert_eq!(envelope.region(), None, "region");
assert_eq!(envelope.service(), None, "service");
assert_eq!(envelope.remote_addr(), None, "remote_addr");
assert!(envelope.client_info().is_none(), "client_info");
assert_eq!(envelope.native_tls_transport(), None, "native_tls_transport");
assert!(envelope.request_context().is_none(), "request_context");
assert!(envelope.body_read_control().is_none(), "body_read_control");
assert!(envelope.verified_sigv4().is_none(), "verified_sigv4");
assert!(envelope.verified_presigned().is_none(), "verified_presigned");
assert!(envelope.req_info().is_none(), "the access phase installs req_info, not the edge");
}
#[tokio::test]
async fn from_gateway_treats_a_peerless_remote_addr_as_absent() {
let server_ctx = ServerContextSlot::new();
let request = list_buckets_request("/")
.extension(None::<RemoteAddr>)
.body(Empty::new())
.expect("static request");
let envelope = envelope_of(request, &server_ctx).await;
assert_eq!(envelope.remote_addr(), None);
assert!(envelope.residual_extensions().is_empty(), "the peerless marker is consumed, not kept");
}
#[test]
fn credentials_carry_the_secret_only_when_the_gateway_handed_it_over() {
let kept = request_credentials("AKIAENVELOPE", None, Some("token")).expect("credentials");
assert_eq!(kept.access_key(), "AKIAENVELOPE");
assert!(kept.secret_key().is_none());
assert_eq!(kept.session_token(), Some("token"));
let handed = request_credentials("AKIAENVELOPE", Some(b"envelope-secret"), None).expect("credentials");
assert_eq!(handed.secret_key(), Some("envelope-secret"));
assert_eq!(handed.session_token(), None);
}
#[test]
fn an_empty_handed_over_secret_is_refused_instead_of_kept() {
let error = request_credentials("AKIAENVELOPE", Some(b""), None).expect_err("empty");
assert_eq!(error.code(), &S3ErrorCode::InternalError);
}
/// ListBuckets does not opt in to the caller's secret, so the gateway keeps it: the handler
/// sees who signed and no secret, and a reader that needs one is refused instead of being
/// handed an empty string to compare with.
#[tokio::test]
async fn an_operation_that_did_not_opt_in_reaches_the_handler_without_the_caller_s_secret() {
let server_ctx = ServerContextSlot::new();
let envelope = envelope_of(signed_list_buckets(), &server_ctx).await;
let credentials = envelope.credentials().expect("a signed request has a caller");
assert_eq!(credentials.access_key(), SIGNER_ACCESS_KEY);
assert!(credentials.secret_key().is_none(), "the secret stays with the gateway");
assert!(credentials.require_secret_key().is_err(), "a reader of the secret fails closed");
assert!(!credentials.secret_key_matches(""), "an absent secret equals no stored secret");
assert!(!credentials.secret_key_matches(SIGNER_SECRET_KEY));
}
#[test]
fn a_secret_that_is_not_text_is_refused_instead_of_dropped() {
let error = request_credentials("AKIAENVELOPE", Some(&[0xff, 0xfe]), None).expect_err("not text");
assert_eq!(error.code(), &S3ErrorCode::InternalError);
}
}
-174
View File
@@ -1,174 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The one conversion of a RustFS error into the gateway's handler error.
//!
//! Responsible for: [`handler_error`], called by every gateway handler and by nothing else.
//! NOT responsible for: choosing an error (the use case's), or writing it (the gateway renders the
//! document, the status and the request identifiers).
//! Upstream: `rustfs_s3_types::S3Error`. Downstream: `rustfs_gateway::HandlerError`.
//!
//! The gateway's error carries a code with its status, a message, and response headers from a
//! closed set. An error this function cannot carry whole is answered `500 InternalError` rather
//! than sent with a member missing: a `416` without its `Content-Range`, or a throttle without
//! its `Retry-After`, tells a client something RustFS did not say. Each operation slice that
//! raises a header not listed in [`error_header`] adds it there, with its test.
use http::{HeaderName, HeaderValue, StatusCode};
use rustfs_gateway::{ErrorHeader, HandlerError};
use rustfs_gateway_types::ErrorCode;
use rustfs_s3_types::S3Error;
use std::borrow::Cow;
/// Answered when an error names a code without a status: the raiser of a custom code must set one.
const MISSING_STATUS: &str = "the error carries a code without a status";
/// Answered when an error carries a response header this edge cannot write.
const UNWRITABLE_HEADER: &str = "the error carries a response header the gateway cannot write";
/// Converts a use-case error into the error the gateway answers with: the same code, status and
/// message, and every response header it carries.
pub(crate) fn handler_error(error: S3Error) -> HandlerError {
let parts = error.into_parts();
let Some(status) = parts.status.or_else(|| parts.code.status_code()) else {
return HandlerError::internal_error(MISSING_STATUS);
};
let mut mapped = HandlerError::new(error_code(parts.code.as_str(), status), parts.message.unwrap_or(Cow::Borrowed("")));
for (name, value) in parts.headers.iter().flatten() {
let Some(header) = error_header(name, value) else {
return HandlerError::internal_error(UNWRITABLE_HEADER);
};
mapped = mapped.with_header(header);
}
mapped
}
/// The gateway's code for `name` answered with `status`: its own declared code when both agree,
/// otherwise a code with exactly the spelling and status RustFS chose.
fn error_code(name: &str, status: StatusCode) -> ErrorCode {
match ErrorCode::known(name) {
Some(declared) if declared.default_status() == status => declared,
_ => ErrorCode::custom(name.to_owned(), status),
}
}
/// The gateway's spelling of one error response header, or `None` for a header it cannot write.
fn error_header(name: &HeaderName, value: &HeaderValue) -> Option<ErrorHeader> {
if name == http::header::RETRY_AFTER {
let seconds = value.to_str().ok()?.parse().ok()?;
return Some(ErrorHeader::RetryAfter { seconds });
}
None
}
#[cfg(test)]
mod tests {
use super::*;
use http::HeaderMap;
use rustfs_s3_types::S3ErrorCode;
#[test]
fn a_declared_code_keeps_its_spelling_status_and_message() {
let mapped = handler_error(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
assert_eq!(mapped.code().as_str(), "AccessDenied");
assert_eq!(mapped.code().default_status(), StatusCode::FORBIDDEN);
assert_eq!(mapped.message(), "Access Denied");
assert!(mapped.headers().is_empty());
}
#[test]
fn an_internal_error_stays_a_server_fault() {
let mapped = handler_error(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
assert_eq!(mapped.code().as_str(), "InternalError");
assert_eq!(mapped.code().default_status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(mapped.message(), "Not init");
}
#[test]
fn an_explicit_status_overrides_the_declared_one() {
let mut error = S3Error::with_message(S3ErrorCode::AccessDenied, "locked");
error.set_status_code(StatusCode::CONFLICT);
let mapped = handler_error(error);
assert_eq!(mapped.code().as_str(), "AccessDenied");
assert_eq!(mapped.code().default_status(), StatusCode::CONFLICT);
}
#[test]
fn a_custom_code_keeps_the_status_its_raiser_set() {
let mut error = S3Error::with_message(S3ErrorCode::Custom("RustFsOnly".into()), "custom");
error.set_status_code(StatusCode::SERVICE_UNAVAILABLE);
let mapped = handler_error(error);
assert_eq!(mapped.code().as_str(), "RustFsOnly");
assert_eq!(mapped.code().default_status(), StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(mapped.message(), "custom");
}
#[test]
fn a_custom_code_without_a_status_is_a_server_fault_not_a_guess() {
let mapped = handler_error(S3Error::with_message(S3ErrorCode::Custom("RustFsOnly".into()), "custom"));
assert_eq!(mapped.code().as_str(), "InternalError");
assert_eq!(mapped.code().default_status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(mapped.message(), MISSING_STATUS);
}
#[test]
fn a_missing_message_is_not_invented() {
let mapped = handler_error(S3Error::new(S3ErrorCode::AccessDenied));
assert_eq!(mapped.code().as_str(), "AccessDenied");
assert_eq!(mapped.message(), "");
}
#[test]
fn a_retry_after_header_is_carried() {
let mut headers = HeaderMap::new();
headers.insert(http::header::RETRY_AFTER, HeaderValue::from_static("7"));
let mut error = S3Error::with_message(S3ErrorCode::SlowDown, "busy");
error.set_headers(headers);
let mapped = handler_error(error);
assert_eq!(mapped.code().as_str(), "SlowDown");
assert_eq!(mapped.headers(), [ErrorHeader::RetryAfter { seconds: 7 }]);
}
#[test]
fn a_header_the_gateway_cannot_write_fails_the_whole_error() {
let mut headers = HeaderMap::new();
headers.insert("x-rustfs-unlisted", HeaderValue::from_static("value"));
let mut error = S3Error::with_message(S3ErrorCode::AccessDenied, "denied");
error.set_headers(headers);
let mapped = handler_error(error);
assert_eq!(mapped.code().as_str(), "InternalError");
assert_eq!(mapped.message(), UNWRITABLE_HEADER);
assert!(mapped.headers().is_empty());
}
#[test]
fn an_unreadable_retry_after_fails_the_whole_error() {
let mut headers = HeaderMap::new();
headers.insert(http::header::RETRY_AFTER, HeaderValue::from_static("soon"));
let mut error = S3Error::with_message(S3ErrorCode::SlowDown, "busy");
error.set_headers(headers);
assert_eq!(handler_error(error).message(), UNWRITABLE_HEADER);
}
}
-68
View File
@@ -1,68 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The S3 operations the gateway stack serves, one file per operation (rustfs/backlog#2734).
//!
//! Responsible for: [`GatewayBackend`], the one backend every operation is implemented on, and
//! [`register`], the explicit list of what is registered. An operation that is not in that list
//! has no handler, and the gateway answers it `501 NotImplemented`.
//! NOT responsible for: assembling the service (`server/gateway_stack.rs`), authenticating or
//! routing a request (the gateway), or any S3 behaviour: a handler converts and delegates, and
//! the RustFS use case it calls is the one the legacy stack calls.
//! Upstream: the gateway's dispatch. Downstream: `crate::app` use cases.
//!
//! # The shape every handler has
//!
//! `<family>/<snake_op>.rs` holds exactly one `impl Handler<Op> for GatewayBackend`, in four
//! steps and nothing else:
//!
//! 1. build the [`RequestEnvelope`](crate::app::RequestEnvelope) with `from_gateway`;
//! 2. run the access phase: [`access::check`], then the operation's own access hook, which is
//! the authoritative authorization and runs exactly where the legacy stack runs it;
//! 3. call the use case with the envelope and the gateway's own input type;
//! 4. map a refusal with [`error::handler_error`], the only place an error is converted.
//!
//! The gateway's `Authorizer` (`server/gateway_stack/authorizer.rs`) is an earlier gate in front
//! of step 2 and never a replacement for it.
pub(crate) mod access;
mod bucket;
mod envelope;
pub(crate) mod error;
use crate::runtime_sources::ServerContextSlot;
use rustfs_gateway::{ServiceBuilder, dto};
use std::sync::Arc;
/// The backend every gateway operation is implemented on: the server whose use cases answer.
pub(crate) struct GatewayBackend {
server_ctx: Arc<ServerContextSlot>,
}
impl GatewayBackend {
pub(crate) fn new(server_ctx: Arc<ServerContextSlot>) -> Self {
Self { server_ctx }
}
pub(crate) fn server_ctx(&self) -> &Arc<ServerContextSlot> {
&self.server_ctx
}
}
/// Registers every operation the gateway stack serves. Each slice adds one line here and one row
/// to the authorization table; the census in `server/gateway_stack/authorizer.rs` holds the two
/// lists equal.
pub(crate) fn register(builder: ServiceBuilder, backend: &Arc<GatewayBackend>) -> ServiceBuilder {
builder.register::<dto::ListBuckets, _>(Arc::clone(backend))
}
-315
View File
@@ -1,315 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The gateway S3 stack of one server: its assembly, and the facts the host hands every request
//! (rustfs/backlog#2734 T2.5).
//!
//! Responsible for: [`GatewayStack::assemble`], the one place the `rustfs-gateway` service is
//! built for RustFS — the RustFS profile, signature verification against RustFS IAM, the RustFS
//! authorizer, explicit deadlines, and the registered handlers — and [`GatewayStack::call`],
//! which attaches the host's facts to a request before the gateway accepts it.
//! NOT responsible for: choosing the stack (`server/stack.rs`), the middleware around it
//! (`server/http.rs`), or any S3 behaviour (`server/gateway_handlers` and the use cases).
//! Upstream: `server/http.rs`. Downstream: `rustfs-gateway`, `rustfs-gateway-adapter`.
//!
//! # What the host keeps
//!
//! RustFS's own layers stay in front of the gateway and the gateway's counterparts stay off
//! (rustfs/backlog#2771): CORS is answered by `ConditionalCorsLayer`, so no CORS source is
//! installed and the gateway writes no CORS header; the SSE-C transport layer runs first and the
//! gateway's own strict gate stays behind it; and the request identifier is the one
//! `ExternalRequestContextLayer` assigned, handed over as the gateway's `HostRequestId`, so an
//! answer and RustFS's logs name a request identically.
// The stack is under construction, so a build without debug assertions must not carry it. The
// message is checked by `scripts/check_gateway_stack_release_boundary.sh`.
#[cfg(not(debug_assertions))]
compile_error!("gateway-stack is restricted to debug builds");
mod authorizer;
use self::authorizer::{AUTHORIZATION_TABLE, OwnedDecisionCore, RequestLine, RustfsAuthorizer};
use crate::runtime_sources::{ServerContextSlot, current_action_credentials};
use crate::server::gateway_handlers::{GatewayBackend, register};
use crate::server::ssec_transport::is_secure_transport;
use crate::shared_types::{NativeTlsTransport, RemoteAddr};
use crate::storage_api::server::http::request_context::RequestContext;
use http::{Extensions, Request, Response};
use rustfs_config::RUSTFS_REGION;
use rustfs_gateway::{
AssemblyError, Authorizer, BoxFuture, ClientAddr, DEFAULT_MAX_BUFFERED_BODY_BYTES, HandlerDeadlineClass,
HandlerDeadlineConfig, HostRequestId, LegacyRustfsVirtualHosts, MintedTraces, RegionSet, RequestBodyDeadlineConfig,
ServiceBuilder, ServiceConfig, SigV4Authenticator, SseConfig, TransportSecurity,
};
use rustfs_gateway_adapter::{IamAdapters, IamKeyLookup, IamPrincipalLookup, RootCredentials, RootCredentialsError, RootSource};
use rustfs_iam::error::Error as IamError;
use rustfs_iam::manager::get_token_signing_key;
use rustfs_iam::store::object::ObjectStore;
use rustfs_iam::sys::IamSys;
use rustfs_policy::auth::UserIdentity;
use rustfs_trusted_proxies::ClientInfo;
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
/// What the assembly takes from the server it is built for.
pub(crate) struct GatewayAssembly {
pub(crate) server_ctx: Arc<ServerContextSlot>,
/// The configured root credentials, checked once here. Requests read the live ones.
pub(crate) root_access_key: String,
pub(crate) root_secret_key: String,
/// The configured region; the default region signs when none is configured.
pub(crate) region: Option<String>,
/// The virtual-hosted-style base domains of the S3 listener, as the legacy stack reads them.
pub(crate) server_domains: Vec<String>,
/// `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT`: the longest silence between two body reads. Zero
/// disables the bound, as it does on the legacy stack.
pub(crate) body_read_timeout: Duration,
/// Whether Keystone authentication is enabled in configuration (`RUSTFS_KEYSTONE_ENABLE`).
pub(crate) keystone_enabled: bool,
}
/// Whether Keystone authentication is switched on in configuration. A Keystone configuration
/// that cannot be read counts as switched on: only a readable configuration can say it is off.
pub(crate) fn keystone_enabled_in_configuration() -> bool {
rustfs_keystone::KeystoneConfig::from_env().map_or(true, |config| config.enable)
}
/// Why the gateway stack could not be assembled. Startup fails with it; the legacy stack is never
/// started in its place.
#[derive(Debug, thiserror::Error)]
pub(crate) enum GatewayAssemblyError {
#[error("the root credentials cannot verify signatures on the gateway stack: {0}")]
RootCredentials(RootCredentialsError),
#[error("region {region:?} cannot be a signing region on the gateway stack")]
Region { region: String },
#[error("the server domains cannot be served by the gateway stack: {0:?}")]
ServerDomains(rustfs_gateway::LegacyDomainError),
#[error("the gateway service refused its assembly: {0:?}")]
Service(AssemblyError),
#[error(
"RUSTFS_S3_STACK=gateway cannot be served while RUSTFS_KEYSTONE_ENABLE is on: the gateway admits a request that \
carries only X-Auth-Token as anonymous while RustFS reads the Keystone identity behind it; Keystone support on the \
gateway stack is rustfs/gateway#1586"
)]
KeystoneEnabled,
}
/// The assembled gateway service.
#[derive(Clone)]
pub(crate) struct GatewayStack {
service: rustfs_gateway::S3Service,
}
impl GatewayStack {
/// Assembles the gateway stack for one server.
///
/// # Errors
///
/// [`GatewayAssemblyError`] naming the input the gateway refused. The secret key is never
/// part of it.
pub(crate) fn assemble(assembly: GatewayAssembly) -> Result<Self, GatewayAssemblyError> {
let iam = Arc::new(ServerIam {
server_ctx: Arc::clone(&assembly.server_ctx),
});
let root = Arc::new(live_root(Arc::clone(&assembly.server_ctx)));
let core = OwnedDecisionCore::new(Arc::clone(&assembly.server_ctx));
let authorizer = RustfsAuthorizer::new(AUTHORIZATION_TABLE, Arc::new(core));
Self::assemble_with(assembly, iam, root, authorizer)
}
fn assemble_with(
assembly: GatewayAssembly,
iam: Arc<dyn IamPrincipalLookup>,
root: Arc<dyn RootSource>,
authorizer: impl Authorizer,
) -> Result<Self, GatewayAssemblyError> {
// The pinned gateway admits a request that carries only `X-Auth-Token` as anonymous and
// runs no authenticator for it, while the RustFS code behind the gateway reads the
// Keystone identity: the request would be authorized as one caller and served as
// another. Refused until the gateway authenticates the token (rustfs/gateway#1586).
if assembly.keystone_enabled {
return Err(GatewayAssemblyError::KeystoneEnabled);
}
// A root the gateway could not verify a signature with is a startup error, never a
// server that silently has no root. Requests read the live root from `root`.
RootCredentials::new(&assembly.root_access_key, &assembly.root_secret_key, &assembly.root_secret_key)
.map_err(GatewayAssemblyError::RootCredentials)?;
let region = assembly.region.as_deref().unwrap_or(RUSTFS_REGION);
let regions = RegionSet::new([region]).map_err(|_| GatewayAssemblyError::Region {
region: region.to_owned(),
})?;
let hosts = LegacyRustfsVirtualHosts::new(&assembly.server_domains).map_err(GatewayAssemblyError::ServerDomains)?;
let backend = Arc::new(GatewayBackend::new(Arc::clone(&assembly.server_ctx)));
// Every legacy RustFS reading of a request, a credential scope and an answer, in the
// gateway's one reviewable preset; its other half is on the authenticator below.
let builder = ServiceBuilder::new().rustfs_profile();
// The credential provider and the policy source go in together, over one IAM and one
// live root, so a key cannot be root to one and an ordinary user to the other.
let builder = IamAdapters::new(iam, root)
.install(builder, |credentials| SigV4Authenticator::new(credentials, regions).rustfs_profile());
let builder = builder
.authorizer(authorizer)
// The outer SSE-C transport layer answers first; the gateway's own gate stays strict
// behind it.
.sse_config(SseConfig::strict())
.host_resolver(hosts)
// The fallback only: `attach_host_facts` hands RustFS's own identifier over.
.trace_source(MintedTraces::with_uuid_request_ids());
let (builder, _settings) = builder.config(service_config(assembly.body_read_timeout));
let service = register(builder, &backend).build().map_err(GatewayAssemblyError::Service)?;
Ok(Self { service })
}
/// The operations the assembled service answers, read from its registry. Everything else is
/// answered `501 NotImplemented`.
pub(crate) fn registered_operations(&self) -> Vec<&'static str> {
self.service.operations().collect()
}
/// Answers one request on the gateway stack, with the host's facts attached.
pub(crate) fn call<B>(&self, mut request: Request<B>) -> impl Future<Output = Response<rustfs_gateway::Body>> + Send + 'static
where
B: http_body::Body + Send + 'static,
B::Data: Send,
B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
{
// The authorizer is handed the headers and the query but not the request line, and the
// shared decision core is asked about the request as it arrived.
let line = RequestLine {
method: request.method().clone(),
uri: request.uri().clone(),
};
request.extensions_mut().insert(line);
attach_host_facts(request.extensions_mut());
let mut service = self.service.clone();
let answer = tower::Service::call(&mut service, request);
async move {
match answer.await {
Ok(response) => response,
Err(never) => match never {},
}
}
}
}
/// The gateway's request settings for RustFS, every deadline stated instead of inherited.
///
/// No handler runs under a total deadline, object data included: the legacy stack has none, and a
/// large transfer must not be cut off for its duration. A committed answer is not bounded either.
/// The request body keeps RustFS's own idle bound and no throughput floor.
fn service_config(body_read_timeout: Duration) -> ServiceConfig {
let handlers = HandlerDeadlineConfig::default()
.without_deadline(HandlerDeadlineClass::Standard)
.without_deadline(HandlerDeadlineClass::Extended)
.without_commit_progress_deadline();
ServiceConfig::new(DEFAULT_MAX_BUFFERED_BODY_BYTES)
.with_handler_deadlines(handlers)
.with_request_body_deadlines(body_deadlines(body_read_timeout))
}
/// RustFS's body read timeout as the gateway's idle deadlines: the same silence is allowed before
/// the first byte and between reads, and zero lifts both.
fn body_deadlines(read_timeout: Duration) -> RequestBodyDeadlineConfig {
RequestBodyDeadlineConfig::new(read_timeout, read_timeout)
.unwrap_or(RequestBodyDeadlineConfig::S3.without_idle_deadlines())
.without_throughput_floor()
}
/// Attaches what the gateway reads about a request from its host: whether the transport was
/// secure, the client address after RustFS's trusted-proxy correction, and RustFS's request
/// identifier. Each is derived from what RustFS's own layers already attached; no header is read.
fn attach_host_facts(extensions: &mut Extensions) {
let client_info = extensions.get::<ClientInfo>();
let native_tls = extensions.get::<NativeTlsTransport>().is_some_and(|transport| transport.0);
let security = if is_secure_transport(native_tls, client_info) {
TransportSecurity::Encrypted
} else {
TransportSecurity::Plaintext
};
let client_ip = client_info.map(|info| info.real_ip).or_else(|| {
extensions
.get::<Option<RemoteAddr>>()
.and_then(|peer| peer.as_ref().map(|peer| peer.0.ip()))
});
// An identifier outside the gateway's alphabet is not handed over; the gateway mints its own.
let request_id = extensions
.get::<RequestContext>()
.and_then(|context| HostRequestId::new(&context.request_id).ok());
extensions.insert(security);
if let Some(client_ip) = client_ip {
extensions.insert(ClientAddr::from_peer(client_ip));
}
if let Some(request_id) = request_id {
extensions.insert(request_id);
}
}
/// The root credentials of one server as they are now, read on every call from where the legacy
/// stack reads them, so a root change reaches both stacks at once. `None` while they are not
/// initialized or cannot verify a signature, which the adapters answer as unavailable.
fn live_root(server_ctx: Arc<ServerContextSlot>) -> impl Fn() -> Option<Arc<RootCredentials>> + Send + Sync + 'static {
move || {
let root = match server_ctx.app_context() {
Some(context) => context.action_credentials().get(),
None => current_action_credentials(),
}?;
// IAM signs STS session tokens with its token-signing key, which is the root secret
// until IAM has published one of its own.
let signing_key = get_token_signing_key().unwrap_or_else(|| root.secret_key.clone());
RootCredentials::new(&root.access_key, &root.secret_key, &signing_key)
.ok()
.map(Arc::new)
}
}
/// The IAM system of one server, looked up when a credential is: IAM is installed after the
/// listener starts, so the assembly cannot hold a handle.
struct ServerIam {
server_ctx: Arc<ServerContextSlot>,
}
impl ServerIam {
fn ready(&self) -> Option<Arc<IamSys<ObjectStore>>> {
self.server_ctx
.app_context()
.filter(|context| context.iam().is_ready())
.map(|context| context.iam().handle())
}
}
impl IamKeyLookup for ServerIam {
fn is_ready(&self) -> bool {
self.ready().is_some()
}
fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
Box::pin(async move {
let Some(iam) = self.ready() else {
return Err(IamError::IamSysNotInitialized);
};
iam.check_key(access_key).await
})
}
}
impl IamPrincipalLookup for ServerIam {
fn group_memberships(&self, user: &str) -> Vec<String> {
self.ready().map(|iam| iam.user_group_memberships(user)).unwrap_or_default()
}
}
#[cfg(test)]
mod tests;
@@ -1,687 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The gateway `Authorizer` of the RustFS assembly: an early gate, never the authorization.
//!
//! Responsible for: [`AUTHORIZATION_TABLE`], one row per registered operation, and
//! [`RustfsAuthorizer`], which answers both gateway authorization stages from it.
//! NOT responsible for: the authoritative decision. That is the legacy access hook in its owned
//! form, run by the operation's handler immediately before its use case
//! (`server/gateway_handlers`), where the legacy stack runs it; decisions, error codes and their
//! order are therefore the same code on both stacks (rustfs/backlog#2761).
//! Upstream: the gateway's route and input authorization stages. Downstream: the shared
//! authorization core, through [`DecisionCore`].
//!
//! # What this gate may do
//!
//! It may refuse a request the hook would certainly refuse too, and nothing else. It is never
//! the only check, and it never allows by default:
//!
//! - an operation without a row is denied;
//! - a `RouteDecidable` row asks the shared decision core the hook asks, with the same facts;
//! - a `DeferToHandlerHook` row evaluates only its [`RouteGate`], which names exactly what the
//! route stage can know, and leaves the rest to the hook;
//! - a `LegacyNoCheck` row is an operation whose legacy hook is empty, said so explicitly;
//! - a core that cannot answer is `Indeterminate`, which the gateway refuses.
//!
//! Plain rows therefore evaluate policy twice, once here and once in the hook. That is accepted
//! until it is measured.
//!
//! A row may be `RouteDecidable` only when its denial concerns the one routed resource and the
//! one routed action. An operation whose denial can concern a second resource (a copy source) or
//! a version-specific action (a versioned read) is a `DeferToHandlerHook` row: the hook writes
//! the denial record with the resource and action it actually refused, and an early refusal
//! here would record the wrong ones.
//!
//! # Deferred rows
//!
//! Every row that is not `RouteDecidable` is listed here, and the census below holds the table
//! to these two lists, so a deferral cannot be added without this section changing.
//!
//! `DeferToHandlerHook`:
//!
//! - `ListBuckets`: the legacy hook is empty and the use case decides per caller. It lists every
//! bucket for a caller allowed `s3:ListAllMyBuckets`, otherwise the buckets the caller may list
//! or locate, and refuses when there is none; that needs the bucket inventory. The route stage
//! refuses an anonymous caller, which the use case refuses with the same answer.
//!
//! `LegacyNoCheck`: none.
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead, authorize_owned_request};
use crate::auth::get_session_token;
use crate::runtime_sources::ServerContextSlot;
use crate::shared_types::RemoteAddr;
use crate::storage_api::server::http::access::resolve_req_info;
use crate::storage_api::server::http::request_context::RequestContext;
use http::{Extensions, HeaderMap, Method, Uri};
use rustfs_gateway::{
Authorizer, AuthzRequest, BoxFuture, Decision, InputAuthzRequest, InputDecisions, RequestContext as GatewayRequestContext,
};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
use rustfs_trusted_proxies::ClientInfo;
use std::net::SocketAddr;
use std::sync::Arc;
/// The resource an operation's action is asked about.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ResourceForm {
/// The service itself: no bucket and no key.
Service,
/// One bucket.
#[cfg_attr(not(test), expect(dead_code, reason = "constructed by the first bucket-level row"))]
Bucket,
/// One object in one bucket.
#[cfg_attr(not(test), expect(dead_code, reason = "constructed by the first object-level row"))]
Object,
}
/// What a deferred row's route stage evaluates: only facts that do not depend on the body.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum RouteGate {
/// Refuses a request that presented no credentials. A request with a verified caller passes
/// to the hook, which decides.
AuthenticatedCaller,
}
/// How much of an operation's decision the route stage can make.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum RouteStage {
/// The route stage asks the shared decision core the same question, with the same facts, as
/// the hook.
#[cfg_attr(not(test), expect(dead_code, reason = "constructed by the first plain operation row"))]
RouteDecidable,
/// The route stage evaluates `gate` and the hook in the handler decides the rest.
DeferToHandlerHook {
/// Exactly what the route stage evaluates.
gate: RouteGate,
/// Why the route stage cannot decide the whole of it.
reason: &'static str,
},
/// The legacy hook for this operation is empty, so there is nothing to evaluate early.
#[cfg_attr(
not(test),
expect(dead_code, reason = "constructed by the row of an operation whose legacy hook is empty")
)]
LegacyNoCheck {
/// Why legacy checks nothing.
reason: &'static str,
},
}
/// The checks an operation's legacy hook performs beyond its one action, which the shared
/// decision core does not perform. There is no default: every row states this column.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum ExtraChecks {
/// The hook asks its one action and nothing else.
#[cfg_attr(not(test), expect(dead_code, reason = "constructed by the first plain operation row"))]
None,
/// The hook, or the use case it defers to, also performs these.
Performed(&'static [&'static str]),
}
/// One operation's authorization, as the legacy access hook performs it.
///
/// A struct literal names every column, so a row cannot leave one unspecified.
#[cfg_attr(
not(test),
expect(
dead_code,
reason = "the body-input and extra-check columns record what the handler hook does and are read by the census"
)
)]
#[derive(Clone, Copy, Debug)]
pub(crate) struct AuthorizationRow {
/// The operation, by the name it is registered under.
pub(crate) operation: &'static str,
/// The action the legacy hook asks, mapped from the operation and not from the gateway's
/// spelling of its action: the two differ for 22 operations, and legacy's is the baseline.
pub(crate) action: Action,
pub(crate) resource: ResourceForm,
pub(crate) route_stage: RouteStage,
/// The body-derived inputs the hook needs, which no route stage has.
pub(crate) body_inputs: &'static [&'static str],
pub(crate) extra_checks: ExtraChecks,
}
/// Every operation the gateway stack registers. Kept equal to the registered set by the census
/// in `server/gateway_stack.rs`.
pub(crate) static AUTHORIZATION_TABLE: &[AuthorizationRow] = &[AuthorizationRow {
operation: "ListBuckets",
action: Action::S3Action(S3Action::ListAllMyBucketsAction),
resource: ResourceForm::Service,
route_stage: RouteStage::DeferToHandlerHook {
gate: RouteGate::AuthenticatedCaller,
reason: "the use case decides per caller from the bucket inventory",
},
body_inputs: &[],
extra_checks: ExtraChecks::Performed(&[
"a caller without s3:ListAllMyBuckets sees the buckets it may list (s3:ListBucket) or locate (s3:GetBucketLocation), \
judged by IAM alone, and is refused when there is none",
]),
}];
/// The request line as it reached RustFS, attached by the host before the gateway accepts the
/// request. The gateway hands an authorizer the headers and the query, but neither the method
/// nor the target, and the shared core is asked about the request as it arrived.
#[derive(Clone, Debug)]
pub(crate) struct RequestLine {
pub(crate) method: Method,
pub(crate) uri: Uri,
}
/// The request facts the shared decision core is asked with, each copied from where the gateway
/// or a RustFS layer in front of it put it.
pub(crate) struct RouteFacts<'a> {
/// The access key the gateway verified; `None` for an anonymous request.
pub(crate) access_key: Option<&'a str>,
/// The bucket, key and version the gateway routed, when the request names them.
pub(crate) bucket: Option<&'a str>,
pub(crate) key: Option<&'a str>,
pub(crate) version_id: Option<&'a str>,
/// The accepted header lines; `None` when the gateway handed over none.
pub(crate) headers: Option<HeaderMap>,
pub(crate) request_line: Option<&'a RequestLine>,
pub(crate) client_info: Option<&'a ClientInfo>,
pub(crate) remote_addr: Option<SocketAddr>,
pub(crate) request_context: Option<&'a RequestContext>,
}
/// The shared authorization decision core, as the route stage of a `RouteDecidable` row asks it.
pub(crate) trait DecisionCore: Send + Sync + 'static {
/// Decides `row`'s action for the caller and resource of `facts`: `Ok` to allow,
/// `AccessDenied` to deny, and any other error when the core could not answer.
fn decide<'a>(&'a self, row: &'a AuthorizationRow, facts: &'a RouteFacts<'a>) -> BoxFuture<'a, S3Result<()>>;
}
/// The decision core both stacks share: the owned authorization entry point, asked about the
/// request the route stage sees.
///
/// The caller is resolved by the function the legacy access check resolves it with, and the
/// decision is `authorize_owned_request`, which the handler's access hook calls too. It is only
/// the policy decision: a row that needs a body-derived input or a composite check is not
/// `RouteDecidable`.
pub(crate) struct OwnedDecisionCore {
server_ctx: Arc<ServerContextSlot>,
}
impl OwnedDecisionCore {
pub(crate) fn new(server_ctx: Arc<ServerContextSlot>) -> Self {
Self { server_ctx }
}
async fn authorize(&self, row: &AuthorizationRow, facts: &RouteFacts<'_>) -> S3Result<()> {
let (Some(line), Some(headers)) = (facts.request_line, facts.headers.as_ref()) else {
return Err(unanswerable("the route stage was handed no request line or headers"));
};
let session_token = get_session_token(&line.uri, headers).unwrap_or_default().to_owned();
let mut req_info =
resolve_req_info(&self.server_ctx, facts.access_key, &session_token, facts.request_context.cloned()).await?;
match row.resource {
ResourceForm::Service => {}
ResourceForm::Bucket => req_info.bucket = Some(routed(facts.bucket)?),
ResourceForm::Object => {
req_info.bucket = Some(routed(facts.bucket)?);
req_info.object = Some(routed(facts.key)?);
req_info.version_id = facts.version_id.map(str::to_owned);
}
}
let credentials = facts.access_key.map(|access_key| {
let session_token = (!session_token.is_empty()).then(|| session_token.clone());
RequestCredentials::without_secret_key(access_key.to_owned(), session_token)
});
let mut extensions = Extensions::new();
extensions.insert(facts.remote_addr.map(RemoteAddr));
if let Some(client_info) = facts.client_info {
extensions.insert(client_info.clone());
}
if let Some(request_context) = facts.request_context {
extensions.insert(request_context.clone());
}
extensions.insert(Arc::clone(&self.server_ctx));
let head = RequestHead {
method: line.method.clone(),
uri: line.uri.clone(),
headers: headers.clone(),
credentials,
region: None,
service: None,
};
let mut envelope = RequestEnvelope::from_parts(head, extensions);
envelope.install_req_info(req_info);
authorize_owned_request(&mut envelope, row.action, None).await
}
}
impl DecisionCore for OwnedDecisionCore {
fn decide<'a>(&'a self, row: &'a AuthorizationRow, facts: &'a RouteFacts<'a>) -> BoxFuture<'a, S3Result<()>> {
Box::pin(self.authorize(row, facts))
}
}
/// The routed name a row's resource form requires. Its absence is a question the core cannot
/// answer, never a decision about some other resource.
fn routed(name: Option<&str>) -> S3Result<String> {
name.map(str::to_owned)
.ok_or_else(|| unanswerable("the route stage was handed no name for the row's resource"))
}
fn unanswerable(message: &'static str) -> S3Error {
S3Error::with_message(S3ErrorCode::InternalError, message)
}
/// What the table decides about one request by itself, and what it leaves to the core.
enum Question<'a> {
Decided(Decision),
AskCore(&'static AuthorizationRow, Box<RouteFacts<'a>>),
}
/// The RustFS implementation of the gateway's two authorization stages.
pub(crate) struct RustfsAuthorizer {
table: &'static [AuthorizationRow],
core: Arc<dyn DecisionCore>,
}
impl RustfsAuthorizer {
/// The authorizer over `table`, asking `core` for `RouteDecidable` rows.
pub(crate) fn new(table: &'static [AuthorizationRow], core: Arc<dyn DecisionCore>) -> Self {
Self { table, core }
}
/// Reads `operation`'s row. `facts` is built only for a row that asks the core.
fn question<'f>(&self, operation: &str, anonymous: bool, facts: impl FnOnce() -> RouteFacts<'f>) -> Question<'f> {
let table: &'static [AuthorizationRow] = self.table;
let Some(row) = table.iter().find(|row| row.operation == operation) else {
// No row, no answer to give: refused, never allowed.
return Question::Decided(Decision::Deny);
};
match row.route_stage {
RouteStage::RouteDecidable => Question::AskCore(row, Box::new(facts())),
RouteStage::DeferToHandlerHook { gate, .. } => Question::Decided(match gate {
RouteGate::AuthenticatedCaller if anonymous => Decision::Deny,
RouteGate::AuthenticatedCaller => Decision::Allow,
}),
RouteStage::LegacyNoCheck { .. } => Question::Decided(Decision::Allow),
}
}
async fn answer(&self, question: Question<'_>) -> Decision {
match question {
Question::Decided(decision) => decision,
Question::AskCore(row, facts) => core_decision(self.core.decide(row, &facts).await),
}
}
fn ask<'a>(&self, context: &GatewayRequestContext<'a>, request: &AuthzRequest<'a>) -> Question<'a> {
self.question(request.operation, request.identity.is_none(), || route_facts(context, request))
}
}
/// Reads the core's answer as a decision. Only an explicit denial is a `Deny`; a failure to
/// answer is `Indeterminate` and never an allowance.
fn core_decision(answer: S3Result<()>) -> Decision {
match answer {
Ok(()) => Decision::Allow,
Err(error) if error.code() == &S3ErrorCode::AccessDenied => Decision::Deny,
Err(_) => Decision::Indeterminate,
}
}
fn route_facts<'a>(context: &GatewayRequestContext<'a>, request: &AuthzRequest<'a>) -> RouteFacts<'a> {
let host = context.server_extensions();
RouteFacts {
access_key: request.identity.map(|identity| identity.access_key_id()),
bucket: request.bucket.map(|bucket| bucket.as_str()),
key: request.key.map(|key| key.as_str()),
version_id: request.version_id,
headers: context.headers().map(|accepted| {
let mut headers = HeaderMap::new();
for (name, value) in accepted.iter_raw() {
headers.append(name.clone(), value.clone());
}
headers
}),
request_line: host.get::<RequestLine>(),
client_info: host.get::<ClientInfo>(),
remote_addr: host
.get::<Option<RemoteAddr>>()
.and_then(|peer| peer.as_ref().map(|peer| peer.0)),
request_context: host.get::<RequestContext>(),
}
}
impl Authorizer for RustfsAuthorizer {
fn authorize_route<'a>(
&'a self,
context: &'a GatewayRequestContext<'a>,
request: &'a AuthzRequest<'a>,
) -> BoxFuture<'a, Decision> {
let question = self.ask(context, request);
Box::pin(self.answer(question))
}
fn authorize_input<'a>(
&'a self,
context: &'a GatewayRequestContext<'a>,
request: &'a InputAuthzRequest<'a>,
) -> BoxFuture<'a, InputDecisions> {
// `decide_all` visits every derived resource and then the visibility question, in that
// order; each is read from the table now and answered below, where the core is awaited.
let stage = self.ask(context, request.route());
let mut derived: Vec<Question<'a>> = request
.resources()
.iter()
.map(|resource| self.ask(context, resource))
.collect();
if let Some(visibility) = request.visibility() {
derived.push(self.ask(context, visibility));
}
Box::pin(async move {
let stage = self.answer(stage).await;
let mut answers = Vec::with_capacity(derived.len());
for question in derived {
answers.push(self.answer(question).await);
}
let mut answers = answers.into_iter();
request.decide_all(stage, |_| answers.next().unwrap_or(Decision::Deny))
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::BTreeSet;
use std::sync::Mutex;
/// The rows documented as `DeferToHandlerHook` in this file's header.
const DOCUMENTED_DEFERRED_ROWS: &[&str] = &["ListBuckets"];
/// The rows documented as `LegacyNoCheck` in this file's header.
const DOCUMENTED_NO_CHECK_ROWS: &[&str] = &[];
static PLAIN_TABLE: &[AuthorizationRow] = &[
AuthorizationRow {
operation: "HeadBucket",
action: Action::S3Action(S3Action::ListBucketAction),
resource: ResourceForm::Bucket,
route_stage: RouteStage::RouteDecidable,
body_inputs: &[],
extra_checks: ExtraChecks::None,
},
AuthorizationRow {
operation: "GetObjectTorrent",
action: Action::S3Action(S3Action::GetObjectAction),
resource: ResourceForm::Object,
route_stage: RouteStage::LegacyNoCheck {
reason: "the legacy hook is empty",
},
body_inputs: &[],
extra_checks: ExtraChecks::None,
},
];
/// A core that gives one scripted answer and records what it was asked.
struct ScriptedCore {
answer: fn() -> S3Result<()>,
asked: Mutex<Vec<(&'static str, Action, Option<String>)>>,
}
impl ScriptedCore {
fn answering(answer: fn() -> S3Result<()>) -> Arc<Self> {
Arc::new(Self {
answer,
asked: Mutex::new(Vec::new()),
})
}
}
impl DecisionCore for ScriptedCore {
fn decide<'a>(&'a self, row: &'a AuthorizationRow, facts: &'a RouteFacts<'a>) -> BoxFuture<'a, S3Result<()>> {
self.asked
.lock()
.expect("core lock")
.push((row.operation, row.action, facts.access_key.map(str::to_owned)));
let answer = (self.answer)();
Box::pin(async move { answer })
}
}
fn denied() -> S3Result<()> {
Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"))
}
fn unavailable() -> S3Result<()> {
Err(S3Error::with_message(S3ErrorCode::InternalError, "the policy store is unavailable"))
}
/// One request as a test asks about it.
struct Asked<'a> {
operation: &'a str,
facts: RouteFacts<'a>,
}
fn facts<'a>(operation: &'a str, access_key: Option<&'a str>) -> Asked<'a> {
Asked {
operation,
facts: RouteFacts {
access_key,
bucket: None,
key: None,
version_id: None,
headers: None,
request_line: None,
client_info: None,
remote_addr: None,
request_context: None,
},
}
}
async fn decision(table: &'static [AuthorizationRow], core: Arc<dyn DecisionCore>, asked: Asked<'_>) -> Decision {
let authorizer = RustfsAuthorizer::new(table, core);
let anonymous = asked.facts.access_key.is_none();
let question = authorizer.question(asked.operation, anonymous, || asked.facts);
authorizer.answer(question).await
}
/// The real decision core of a server that has no object store yet.
fn owned_core() -> Arc<dyn DecisionCore> {
Arc::new(OwnedDecisionCore::new(ServerContextSlot::new()))
}
fn head_bucket() -> RequestLine {
RequestLine {
method: Method::HEAD,
uri: Uri::from_static("/bucket"),
}
}
fn rows_where(predicate: fn(&RouteStage) -> bool) -> BTreeSet<&'static str> {
AUTHORIZATION_TABLE
.iter()
.filter(|row| predicate(&row.route_stage))
.map(|row| row.operation)
.collect()
}
#[tokio::test]
async fn an_operation_without_a_row_is_denied() {
let core = ScriptedCore::answering(|| Ok(()));
for operation in ["GetObject", "PutObject", "", "listbuckets"] {
let decided = decision(AUTHORIZATION_TABLE, core.clone(), facts(operation, Some("AKIDUSER0001"))).await;
assert_eq!(decided, Decision::Deny, "{operation:?}");
}
assert!(core.asked.lock().expect("core lock").is_empty(), "an unmapped operation asks nothing");
}
#[tokio::test]
async fn a_core_that_cannot_answer_is_indeterminate_never_an_allowance() {
let decided = decision(
PLAIN_TABLE,
ScriptedCore::answering(unavailable),
facts("HeadBucket", Some("AKIDUSER0001")),
)
.await;
assert_eq!(decided, Decision::Indeterminate);
}
#[tokio::test]
async fn the_owned_core_without_a_store_cannot_answer_and_never_allows() {
let line = head_bucket();
let mut asked = facts("HeadBucket", None);
asked.facts.bucket = Some("bucket");
asked.facts.headers = Some(HeaderMap::new());
asked.facts.request_line = Some(&line);
assert_eq!(decision(PLAIN_TABLE, owned_core(), asked).await, Decision::Indeterminate);
}
#[tokio::test]
async fn the_owned_core_refuses_a_request_the_host_did_not_describe() {
let line = head_bucket();
let mut without_a_line = facts("HeadBucket", None);
without_a_line.facts.bucket = Some("bucket");
without_a_line.facts.headers = Some(HeaderMap::new());
assert_eq!(decision(PLAIN_TABLE, owned_core(), without_a_line).await, Decision::Indeterminate);
let mut without_headers = facts("HeadBucket", None);
without_headers.facts.bucket = Some("bucket");
without_headers.facts.request_line = Some(&line);
assert_eq!(decision(PLAIN_TABLE, owned_core(), without_headers).await, Decision::Indeterminate);
}
#[tokio::test]
async fn the_owned_core_refuses_a_bucket_row_whose_request_names_no_bucket() {
let line = head_bucket();
let mut asked = facts("HeadBucket", None);
asked.facts.headers = Some(HeaderMap::new());
asked.facts.request_line = Some(&line);
assert_eq!(decision(PLAIN_TABLE, owned_core(), asked).await, Decision::Indeterminate);
}
#[tokio::test]
async fn a_core_denial_is_a_denial() {
let decided = decision(PLAIN_TABLE, ScriptedCore::answering(denied), facts("HeadBucket", Some("AKIDUSER0001"))).await;
assert_eq!(decided, Decision::Deny);
}
#[tokio::test]
async fn a_route_decidable_row_asks_the_core_its_legacy_action_for_the_verified_caller() {
let core = ScriptedCore::answering(|| Ok(()));
let decided = decision(PLAIN_TABLE, core.clone(), facts("HeadBucket", Some("AKIDUSER0001"))).await;
assert_eq!(decided, Decision::Allow);
assert_eq!(
*core.asked.lock().expect("core lock"),
[(
"HeadBucket",
Action::S3Action(S3Action::ListBucketAction),
Some("AKIDUSER0001".to_owned())
)]
);
}
#[tokio::test]
async fn a_deferred_row_refuses_an_anonymous_caller_without_asking_the_core() {
let core = ScriptedCore::answering(|| Ok(()));
assert_eq!(
decision(AUTHORIZATION_TABLE, core.clone(), facts("ListBuckets", None)).await,
Decision::Deny
);
assert_eq!(
decision(AUTHORIZATION_TABLE, core.clone(), facts("ListBuckets", Some("AKIDUSER0001"))).await,
Decision::Allow,
"a verified caller passes to the hook"
);
assert!(
core.asked.lock().expect("core lock").is_empty(),
"a deferred row decides nothing through the core"
);
}
#[tokio::test]
async fn a_no_check_row_is_allowed_only_because_its_row_says_so() {
let core = ScriptedCore::answering(denied);
assert_eq!(
decision(PLAIN_TABLE, core.clone(), facts("GetObjectTorrent", None)).await,
Decision::Allow
);
assert_eq!(decision(AUTHORIZATION_TABLE, core, facts("GetObjectTorrent", None)).await, Decision::Deny);
}
#[test]
fn the_deferred_rows_are_exactly_the_documented_ones() {
let deferred = rows_where(|stage| matches!(stage, RouteStage::DeferToHandlerHook { .. }));
assert_eq!(deferred, DOCUMENTED_DEFERRED_ROWS.iter().copied().collect::<BTreeSet<_>>());
}
#[test]
fn the_no_check_rows_are_exactly_the_documented_ones() {
let unchecked = rows_where(|stage| matches!(stage, RouteStage::LegacyNoCheck { .. }));
assert_eq!(unchecked, DOCUMENTED_NO_CHECK_ROWS.iter().copied().collect::<BTreeSet<_>>());
}
#[test]
fn every_row_states_every_column_and_names_one_operation() {
let mut seen = BTreeSet::new();
for row in AUTHORIZATION_TABLE {
assert!(seen.insert(row.operation), "{} has two rows", row.operation);
assert!(!row.operation.is_empty());
match row.route_stage {
RouteStage::DeferToHandlerHook { reason, .. } | RouteStage::LegacyNoCheck { reason } => {
assert!(!reason.is_empty(), "{} defers without a reason", row.operation);
}
RouteStage::RouteDecidable => {}
}
match row.extra_checks {
ExtraChecks::Performed(checks) => {
assert!(!checks.is_empty(), "{} lists no extra check; say `ExtraChecks::None`", row.operation);
assert!(checks.iter().all(|check| !check.is_empty()), "{}", row.operation);
}
ExtraChecks::None => {}
}
assert!(row.body_inputs.iter().all(|input| !input.is_empty()), "{}", row.operation);
}
}
#[test]
fn list_buckets_asks_the_legacy_action_on_the_service() {
let row = AUTHORIZATION_TABLE
.iter()
.find(|row| row.operation == "ListBuckets")
.expect("ListBuckets is registered");
assert_eq!(row.action, Action::S3Action(S3Action::ListAllMyBucketsAction));
assert_eq!(row.resource, ResourceForm::Service);
assert_eq!(
row.route_stage,
RouteStage::DeferToHandlerHook {
gate: RouteGate::AuthenticatedCaller,
reason: "the use case decides per caller from the bucket inventory",
}
);
assert!(row.body_inputs.is_empty());
assert!(matches!(row.extra_checks, ExtraChecks::Performed(_)));
}
}
-561
View File
@@ -1,561 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The assembled gateway stack, driven through [`GatewayStack::call`] exactly as `server/http.rs`
//! drives it. Requests that need object storage are in `rustfs/tests/embedded_gateway_stack_test.rs`.
use super::authorizer::AUTHORIZATION_TABLE;
use super::*;
use bytes::Bytes;
use http::{HeaderMap, Method, StatusCode};
use http_body_util::{BodyExt, Empty};
use rustfs_gateway::{AuthzRequest, ClientFacts, Decision, InputAuthzRequest, InputDecisions, NO_DEADLINE};
use std::collections::BTreeSet;
use std::net::{IpAddr, SocketAddr};
use std::sync::Mutex;
use std::time::Instant;
const ROOT_ACCESS_KEY: &str = "rustfsgatewayroot";
const ROOT_SECRET_KEY: &str = "rustfs-gateway-root-secret";
const REQUEST_ID: &str = "7c9e6679-7425-40de-944b-e07fc1f90ae7";
const PEER: ([u8; 4], u16) = ([10, 0, 0, 7], 41000);
const CLIENT: [u8; 4] = [203, 0, 113, 9];
const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
fn assembly() -> GatewayAssembly {
GatewayAssembly {
server_ctx: ServerContextSlot::new(),
root_access_key: ROOT_ACCESS_KEY.to_owned(),
root_secret_key: ROOT_SECRET_KEY.to_owned(),
region: Some("us-east-1".to_owned()),
server_domains: Vec::new(),
body_read_timeout: Duration::from_secs(300),
keystone_enabled: false,
}
}
/// An IAM whose readiness and answer are scripted.
struct ScriptedIam {
ready: bool,
}
impl IamKeyLookup for ScriptedIam {
fn is_ready(&self) -> bool {
self.ready
}
fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
Box::pin(async { Ok((None, false)) })
}
}
impl IamPrincipalLookup for ScriptedIam {
fn group_memberships(&self, _user: &str) -> Vec<String> {
Vec::new()
}
}
/// A settable live root, standing in for the server's runtime credentials.
#[derive(Clone)]
struct LiveRoot(Arc<Mutex<Option<Arc<RootCredentials>>>>);
impl LiveRoot {
fn set(&self, root: RootCredentials) {
*self.0.lock().expect("root lock") = Some(Arc::new(root));
}
fn clear(&self) {
*self.0.lock().expect("root lock") = None;
}
fn source(&self) -> Arc<dyn RootSource> {
let cell = Arc::clone(&self.0);
Arc::new(move || cell.lock().expect("root lock").clone())
}
}
/// A live root holding the test root.
fn root() -> LiveRoot {
let root = RootCredentials::new(ROOT_ACCESS_KEY, ROOT_SECRET_KEY, ROOT_SECRET_KEY).expect("a usable root");
LiveRoot(Arc::new(Mutex::new(Some(Arc::new(root)))))
}
/// The authorizer production installs: the registered table over the shared decision core.
fn production_authorizer() -> RustfsAuthorizer {
RustfsAuthorizer::new(AUTHORIZATION_TABLE, Arc::new(OwnedDecisionCore::new(ServerContextSlot::new())))
}
/// The stack as production assembles it, over an IAM that knows no key but the root's.
fn stack_over(iam: ScriptedIam) -> GatewayStack {
let authorizer = production_authorizer();
GatewayStack::assemble_with(assembly(), Arc::new(iam), root().source(), authorizer).expect("the stack assembles")
}
fn request(method: Method, target: &str) -> http::request::Builder {
Request::builder().method(method).uri(target).header("host", "localhost")
}
/// A well-formed SigV4 header signature for `access_key` whose signature is not the right one.
fn signed_with_a_wrong_signature(builder: http::request::Builder, access_key: &str) -> http::request::Builder {
let now = jiff::Timestamp::now().strftime("%Y%m%dT%H%M%SZ").to_string();
let date = &now[..8];
builder
.header("x-amz-date", now.as_str())
.header("x-amz-content-sha256", EMPTY_SHA256)
.header(
"authorization",
format!(
"AWS4-HMAC-SHA256 Credential={access_key}/{date}/us-east-1/s3/aws4_request, \
SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature={}",
"0".repeat(64)
),
)
}
async fn answer(stack: &GatewayStack, builder: http::request::Builder) -> (StatusCode, HeaderMap, String) {
let response = stack.call(builder.body(Empty::<Bytes>::new()).expect("static request")).await;
let (parts, body) = response.into_parts();
let body = body.collect().await.expect("the answer is read").to_bytes();
(parts.status, parts.headers, String::from_utf8_lossy(&body).into_owned())
}
fn rustfs_request_context(request_id: &str) -> RequestContext {
RequestContext {
request_id: request_id.to_owned(),
x_amz_request_id: request_id.to_owned(),
trace_id: None,
span_id: None,
start_time: Instant::now(),
}
}
#[test]
fn every_registered_operation_has_an_authorization_row_and_every_row_an_operation() {
let stack = stack_over(ScriptedIam { ready: true });
let registered: BTreeSet<&str> = stack.registered_operations().into_iter().collect();
let mapped: BTreeSet<&str> = AUTHORIZATION_TABLE.iter().map(|row| row.operation).collect();
assert_eq!(registered, mapped);
assert_eq!(registered, BTreeSet::from(["ListBuckets"]), "the walking skeleton serves one operation");
}
#[test]
fn production_assembly_registers_what_the_test_assembly_does() {
let stack = GatewayStack::assemble(assembly()).expect("the stack assembles");
assert_eq!(stack.registered_operations(), ["ListBuckets"]);
}
#[test]
fn an_unusable_region_fails_the_assembly() {
let refused = GatewayStack::assemble(GatewayAssembly {
region: Some("Not A Region".to_owned()),
..assembly()
});
assert!(matches!(refused, Err(GatewayAssemblyError::Region { .. })), "{:?}", refused.err());
}
#[test]
fn an_unusable_root_access_key_fails_the_assembly_without_naming_the_secret() {
let refused = GatewayStack::assemble(GatewayAssembly {
root_access_key: String::new(),
..assembly()
});
let error = refused.err().expect("an empty access key verifies nothing");
assert!(
matches!(error, GatewayAssemblyError::RootCredentials(RootCredentialsError::InvalidAccessKey)),
"{error:?}"
);
for rendered in [error.to_string(), format!("{error:?}")] {
assert!(!rendered.contains(ROOT_SECRET_KEY), "{rendered}");
}
}
#[test]
fn a_missing_or_short_root_secret_fails_the_assembly() {
for weak in ["", "1234567"] {
let refused = GatewayStack::assemble(GatewayAssembly {
root_secret_key: weak.to_owned(),
..assembly()
});
assert!(
matches!(
refused.as_ref().err(),
Some(GatewayAssemblyError::RootCredentials(RootCredentialsError::WeakSecretKey))
),
"{} bytes",
weak.len()
);
}
}
#[tokio::test]
async fn a_root_change_reaches_the_next_request_and_a_withdrawn_root_signs_nothing() {
const ROTATED_ACCESS_KEY: &str = "rotatedgatewayroot";
let live = root();
let authorizer = production_authorizer();
let stack = GatewayStack::assemble_with(assembly(), Arc::new(ScriptedIam { ready: true }), live.source(), authorizer)
.expect("the stack assembles");
let signed_as = |access_key: &str| signed_with_a_wrong_signature(request(Method::GET, "/"), access_key);
let (_, _, known) = answer(&stack, signed_as(ROOT_ACCESS_KEY)).await;
assert!(known.contains("<Code>SignatureDoesNotMatch</Code>"), "the root key is known: {known}");
// The root access key changes: the old one is now just an unknown key, and the new one is
// known to the very next request.
live.set(RootCredentials::new(ROTATED_ACCESS_KEY, ROOT_SECRET_KEY, ROOT_SECRET_KEY).expect("a usable root"));
let (status, _, retired) = answer(&stack, signed_as(ROOT_ACCESS_KEY)).await;
assert_eq!(status, StatusCode::FORBIDDEN, "{retired}");
assert!(retired.contains("<Code>InvalidAccessKeyId</Code>"), "{retired}");
let (_, _, rotated) = answer(&stack, signed_as(ROTATED_ACCESS_KEY)).await;
assert!(
rotated.contains("<Code>SignatureDoesNotMatch</Code>"),
"the new root key is known: {rotated}"
);
// Injected: a root source that returns nothing, so the provider returns an error for the
// lookup. The key that was root a moment ago verifies nothing; the pinned gateway answers the
// provider fault as an unknown key. Ledger entry R-068 (rustfs/backlog#2684): this status
// becomes `500` when the pin includes rustfs/gateway#1585 and the assembly sets its switch.
live.clear();
let (status, _, withdrawn) = answer(&stack, signed_as(ROTATED_ACCESS_KEY)).await;
assert_eq!(status, StatusCode::FORBIDDEN, "{withdrawn}");
assert!(withdrawn.contains("<Code>InvalidAccessKeyId</Code>"), "{withdrawn}");
}
#[test]
fn keystone_enabled_fails_the_assembly_and_names_both_settings() {
let refused = GatewayStack::assemble(GatewayAssembly {
keystone_enabled: true,
..assembly()
});
let error = refused.err().expect("the gateway stack is not served with Keystone enabled");
assert!(matches!(error, GatewayAssemblyError::KeystoneEnabled), "{error:?}");
let message = error.to_string();
for named in ["RUSTFS_S3_STACK=gateway", "RUSTFS_KEYSTONE_ENABLE", "rustfs/gateway#1586"] {
assert!(message.contains(named), "{named} is missing from: {message}");
}
}
#[test]
fn keystone_disabled_assembles() {
GatewayStack::assemble(GatewayAssembly {
keystone_enabled: false,
..assembly()
})
.expect("the gateway stack assembles without Keystone");
}
#[test]
fn keystone_is_read_from_configuration_and_an_unreadable_one_counts_as_enabled() {
const ENABLE: &str = "RUSTFS_KEYSTONE_ENABLE";
const AUTH_URL: &str = "RUSTFS_KEYSTONE_AUTH_URL";
temp_env::with_vars([(ENABLE, None::<&str>), (AUTH_URL, None)], || {
assert!(!keystone_enabled_in_configuration(), "unset is off");
});
temp_env::with_vars([(ENABLE, Some("false")), (AUTH_URL, None)], || {
assert!(!keystone_enabled_in_configuration(), "false is off");
});
temp_env::with_vars([(ENABLE, Some("true")), (AUTH_URL, Some("http://keystone.example.test:5000"))], || {
assert!(keystone_enabled_in_configuration(), "true is on");
});
temp_env::with_vars([(ENABLE, Some("true")), (AUTH_URL, None)], || {
assert!(keystone_enabled_in_configuration(), "on but unreadable is still on");
});
}
#[test]
fn no_handler_runs_under_a_total_deadline_and_the_body_keeps_the_rustfs_idle_bound() {
let config = service_config(Duration::from_secs(300));
assert_eq!(config.handler_deadline(HandlerDeadlineClass::Standard), NO_DEADLINE);
assert_eq!(config.handler_deadline(HandlerDeadlineClass::Extended), NO_DEADLINE);
assert_eq!(config.commit_progress_deadline(), NO_DEADLINE);
assert_eq!(config.request_body_deadlines().read_idle(), Duration::from_secs(300));
assert_eq!(config.request_body_deadlines().first_byte(), Duration::from_secs(300));
assert_eq!(config.request_body_deadlines().throughput_window(), NO_DEADLINE);
}
#[test]
fn a_disabled_body_read_timeout_lifts_the_idle_bound_instead_of_arming_zero() {
let deadlines = body_deadlines(Duration::ZERO);
assert_eq!(deadlines.read_idle(), NO_DEADLINE);
assert_eq!(deadlines.first_byte(), NO_DEADLINE);
}
#[test]
fn host_facts_come_from_what_rustfs_layers_attached() {
let peer = SocketAddr::from(PEER);
let mut extensions = Extensions::new();
extensions.insert(Some(RemoteAddr(peer)));
extensions.insert(ClientInfo::direct(SocketAddr::new(IpAddr::from(CLIENT), 443)));
extensions.insert(NativeTlsTransport(true));
extensions.insert(rustfs_request_context(REQUEST_ID));
attach_host_facts(&mut extensions);
assert_eq!(extensions.get::<TransportSecurity>(), Some(&TransportSecurity::Encrypted));
assert_eq!(
extensions.get::<ClientAddr>().map(|address| address.ip()),
Some(IpAddr::from(CLIENT)),
"the trusted-proxy-corrected address wins over the socket peer"
);
assert_eq!(
extensions
.get::<HostRequestId>()
.map(|id| id.request_id().as_str().to_owned()),
Some(REQUEST_ID.to_owned())
);
}
#[test]
fn host_facts_never_lean_towards_secure_or_invent_an_address() {
let mut bare = Extensions::new();
attach_host_facts(&mut bare);
assert_eq!(bare.get::<TransportSecurity>(), Some(&TransportSecurity::Plaintext));
assert!(bare.get::<ClientAddr>().is_none(), "no address is known");
assert!(bare.get::<HostRequestId>().is_none(), "no identifier is known");
let mut plaintext = Extensions::new();
plaintext.insert(Some(RemoteAddr(SocketAddr::from(PEER))));
plaintext.insert(NativeTlsTransport(false));
plaintext.insert(rustfs_request_context("not a request id"));
attach_host_facts(&mut plaintext);
assert_eq!(plaintext.get::<TransportSecurity>(), Some(&TransportSecurity::Plaintext));
assert_eq!(
plaintext.get::<ClientAddr>().map(|address| address.ip()),
Some(IpAddr::from(PEER.0)),
"without a proxy the socket peer is the client"
);
assert!(
plaintext.get::<HostRequestId>().is_none(),
"an identifier outside the alphabet is not handed over"
);
let mut peerless = Extensions::new();
peerless.insert(None::<RemoteAddr>);
attach_host_facts(&mut peerless);
assert!(peerless.get::<ClientAddr>().is_none());
}
/// Records what the gateway hands its authorizer about the client and the request line, and
/// refuses.
struct RecordingAuthorizer {
seen: Arc<Mutex<Option<Option<ClientFacts>>>>,
line: Arc<Mutex<Option<RequestLine>>>,
}
impl Authorizer for RecordingAuthorizer {
fn authorize_route<'a>(
&'a self,
context: &'a rustfs_gateway::RequestContext<'a>,
_request: &'a AuthzRequest<'a>,
) -> BoxFuture<'a, Decision> {
*self.seen.lock().expect("recording lock") = Some(context.client().copied());
*self.line.lock().expect("recording lock") = context.server_extensions().get::<RequestLine>().cloned();
Box::pin(async { Decision::Deny })
}
fn authorize_input<'a>(
&'a self,
_context: &'a rustfs_gateway::RequestContext<'a>,
request: &'a InputAuthzRequest<'a>,
) -> BoxFuture<'a, InputDecisions> {
let decisions = request.decide_all(Decision::Deny, |_| Decision::Deny);
Box::pin(async move { decisions })
}
}
#[tokio::test]
async fn the_authorizer_sees_the_client_facts_the_host_attached() {
let seen = Arc::new(Mutex::new(None));
let line = Arc::new(Mutex::new(None));
let authorizer = RecordingAuthorizer {
seen: Arc::clone(&seen),
line: Arc::clone(&line),
};
let stack = GatewayStack::assemble_with(assembly(), Arc::new(ScriptedIam { ready: true }), root().source(), authorizer)
.expect("assembles");
let proxied = request(Method::GET, "/?prefix=probe")
.extension(Some(RemoteAddr(SocketAddr::from(PEER))))
.extension(ClientInfo::direct(SocketAddr::new(IpAddr::from(CLIENT), 443)))
.extension(NativeTlsTransport(true));
let (status, _, _) = answer(&stack, proxied).await;
assert_eq!(status, StatusCode::FORBIDDEN);
let facts = seen
.lock()
.expect("recording lock")
.take()
.expect("the authorizer was asked")
.expect("the host attached client facts");
assert!(facts.transport_secure, "the listener terminated TLS");
assert_eq!(facts.client_ip, Some(IpAddr::from(CLIENT)));
assert_eq!(facts.source_ip(), Some(IpAddr::from(CLIENT)));
let line = line
.lock()
.expect("recording lock")
.take()
.expect("the host attached the request line");
assert_eq!(line.method, Method::GET);
assert_eq!(line.uri.path(), "/");
assert_eq!(line.uri.query(), Some("prefix=probe"));
}
#[tokio::test]
async fn the_authorizer_sees_a_cleartext_transport_as_cleartext() {
let seen = Arc::new(Mutex::new(None));
let line = Arc::new(Mutex::new(None));
let authorizer = RecordingAuthorizer {
seen: Arc::clone(&seen),
line: Arc::clone(&line),
};
let stack = GatewayStack::assemble_with(assembly(), Arc::new(ScriptedIam { ready: true }), root().source(), authorizer)
.expect("assembles");
let (status, _, _) = answer(&stack, request(Method::GET, "/").extension(NativeTlsTransport(false))).await;
assert_eq!(status, StatusCode::FORBIDDEN);
let facts = seen
.lock()
.expect("recording lock")
.take()
.expect("the authorizer was asked")
.expect("the transport declaration alone makes the facts present");
assert!(!facts.transport_secure);
assert_eq!(facts.client_ip, None);
}
#[tokio::test]
async fn an_anonymous_list_buckets_is_denied() {
let stack = stack_over(ScriptedIam { ready: true });
let (status, _, body) = answer(&stack, request(Method::GET, "/")).await;
assert_eq!(status, StatusCode::FORBIDDEN);
assert!(body.contains("<Code>AccessDenied</Code>"), "{body}");
}
#[tokio::test]
async fn an_unregistered_operation_is_not_implemented() {
let stack = stack_over(ScriptedIam { ready: true });
for (method, target) in [
(Method::GET, "/bucket/key"),
(Method::PUT, "/bucket"),
(Method::DELETE, "/bucket/key"),
] {
let (status, _, body) = answer(&stack, request(method.clone(), target)).await;
assert_eq!(status, StatusCode::NOT_IMPLEMENTED, "{method} {target}: {body}");
assert!(body.contains("<Code>NotImplemented</Code>"), "{method} {target}: {body}");
}
}
#[tokio::test]
async fn a_wrong_signature_is_a_signature_mismatch() {
let stack = stack_over(ScriptedIam { ready: true });
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), ROOT_ACCESS_KEY)).await;
assert_eq!(status, StatusCode::FORBIDDEN);
assert!(body.contains("<Code>SignatureDoesNotMatch</Code>"), "{body}");
assert!(!body.contains(ROOT_SECRET_KEY), "{body}");
}
#[tokio::test]
async fn an_unknown_access_key_is_refused_as_the_caller_s_fault() {
let stack = stack_over(ScriptedIam { ready: true });
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), "AKIDUNKNOWN0001")).await;
assert_eq!(status, StatusCode::FORBIDDEN, "{body}");
assert!(body.contains("<Code>InvalidAccessKeyId</Code>"), "{body}");
}
/// Injected: a credential provider that returns an error for the lookup of a non-root key.
///
/// The pinned gateway's SigV4 authenticator answers a provider fault exactly as it answers an
/// unknown key, `403 InvalidAccessKeyId`; the legacy stack answers a failed lookup `500`. Ledger
/// entry R-068 (rustfs/backlog#2684): rustfs/gateway#1585 adds an opt-in switch that answers the
/// fault as one. When the pin includes it and the assembly sets it, this test expects `500`.
/// Until then the status is pinned, and what holds on both stacks is that nothing is served.
#[tokio::test]
async fn a_credential_lookup_that_fails_is_refused_and_never_served() {
let stack = stack_over(ScriptedIam { ready: false });
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), "AKIDUSER0001")).await;
assert_eq!(status, StatusCode::FORBIDDEN, "{body}");
assert!(body.contains("<Code>InvalidAccessKeyId</Code>"), "{body}");
assert!(!body.contains("ListAllMyBucketsResult"), "{body}");
}
#[tokio::test]
async fn the_root_key_verifies_while_iam_is_unavailable() {
let stack = stack_over(ScriptedIam { ready: false });
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), ROOT_ACCESS_KEY)).await;
assert_eq!(status, StatusCode::FORBIDDEN, "the root secret is known without IAM: {body}");
assert!(body.contains("<Code>SignatureDoesNotMatch</Code>"), "{body}");
}
#[tokio::test]
async fn the_answer_carries_the_request_identifier_rustfs_assigned() {
let stack = stack_over(ScriptedIam { ready: true });
let (_, headers, _) = answer(&stack, request(Method::GET, "/").extension(rustfs_request_context(REQUEST_ID))).await;
assert_eq!(headers.get("x-amz-request-id").and_then(|value| value.to_str().ok()), Some(REQUEST_ID));
assert_eq!(headers.get("x-request-id").and_then(|value| value.to_str().ok()), Some(REQUEST_ID));
}
#[tokio::test]
async fn without_a_rustfs_identifier_the_gateway_mints_its_own() {
let stack = stack_over(ScriptedIam { ready: true });
let (_, headers, _) = answer(&stack, request(Method::GET, "/")).await;
let minted = headers
.get("x-amz-request-id")
.and_then(|value| value.to_str().ok())
.expect("every answer names its request");
assert!(!minted.is_empty());
assert_ne!(minted, REQUEST_ID);
}
#[tokio::test]
async fn the_gateway_writes_no_cors_header() {
let stack = stack_over(ScriptedIam { ready: true });
let browser = |builder: http::request::Builder| builder.header("origin", "https://app.example.test");
for (method, target) in [(Method::GET, "/"), (Method::GET, "/bucket/key"), (Method::PUT, "/bucket/key")] {
let (_, headers, _) = answer(&stack, browser(request(method.clone(), target))).await;
let cors: Vec<&str> = headers
.keys()
.map(http::HeaderName::as_str)
.filter(|name| name.starts_with("access-control-"))
.collect();
assert!(cors.is_empty(), "{method} {target}: {cors:?}");
}
}
+1 -82
View File
@@ -18,9 +18,6 @@ use crate::app::object::request_body::{BodyReadControl, ObservedBody};
use crate::auth::IAMAuth;
use crate::auth_keystone;
use crate::config;
#[cfg(feature = "gateway-stack")]
use crate::server::gateway_stack::{GatewayAssembly, GatewayStack, keystone_enabled_in_configuration};
use crate::server::stack::{S3StackService, select_stack};
use crate::server::{
Http12ShutdownOutcome, ReadinessGateLayer, RemoteAddr, ShutdownHandle,
compress::{HttpCompressionConfig, PathAwareHttpCompressionPredicate, PathCategoryInjectionLayer},
@@ -262,7 +259,6 @@ const EVENT_HTTP_TRANSPORT_FAILED: &str = "http_transport_failed";
const EVENT_SOCKET_FALLBACK: &str = "socket_fallback";
const EVENT_HTTP_BIND_FAILED: &str = "http_bind_failed";
const EVENT_HTTP_STARTUP_ENDPOINTS: &str = "http_startup_endpoints";
const EVENT_S3_STACK_SELECTED: &str = "s3_stack_selected";
const EVENT_HTTP_HOST_ROUTING: &str = "http_host_routing";
const EVENT_HTTP_COMPRESSION_STATE: &str = "http_compression_state";
const EVENT_API_RATE_LIMIT_STATE: &str = "api_rate_limit_state";
@@ -1464,41 +1460,6 @@ fn control_request_body(body: s3s::Body) -> ControlRequestBody {
ControlRequestBody(body)
}
/// The S3 service mounted under the middleware: the legacy stack or the gateway stack, selected
/// at startup (`server/stack.rs`).
type S3EdgeService = S3StackService<S3Service, GatewayEdgeService>;
/// A build without the `gateway-stack` feature has no gateway stack to select, so this side of
/// [`S3EdgeService`] is never constructed.
#[cfg(not(feature = "gateway-stack"))]
type GatewayEdgeService = S3Service;
/// The gateway stack where the legacy service is mounted: the same response and error types, so
/// the layers around it do not change with the stack.
#[cfg(feature = "gateway-stack")]
#[derive(Clone)]
struct GatewayEdgeService(GatewayStack);
#[cfg(feature = "gateway-stack")]
impl<B> Service<HttpRequest<B>> for GatewayEdgeService
where
B: http_body::Body<Data = Bytes> + Send + 'static,
B::Error: std::error::Error + Send + Sync + 'static,
{
type Response = Response<s3s::Body>;
type Error = s3s::HttpError;
type Future = Pin<Box<dyn Future<Output = std::result::Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, request: HttpRequest<B>) -> Self::Future {
let answer = self.0.call(request);
Box::pin(async move { Ok(answer.await.map(s3s::Body::http_body_unsync)) })
}
}
#[derive(Clone)]
struct ControlBodyDigestService<S> {
inner: S,
@@ -2491,48 +2452,6 @@ async fn start_http_server_inner(
rustfs_config::ENV_HTTP_REQUEST_BODY_READ_TIMEOUT,
rustfs_config::DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT,
));
// The S3 stack is selected once, for the whole server. A gateway stack that cannot be
// assembled fails startup here; the legacy stack is never started in its place.
#[cfg(feature = "gateway-stack")]
let assemble_gateway = || -> Result<GatewayEdgeService> {
let stack = GatewayStack::assemble(GatewayAssembly {
server_ctx: Arc::clone(&server_ctx),
root_access_key: config.access_key.clone(),
root_secret_key: config.secret_key.clone(),
region: config.region.clone(),
server_domains: vh_domain_hosts.to_vec(),
body_read_timeout: http_request_body_read_timeout,
keystone_enabled: keystone_enabled_in_configuration(),
})
.map_err(Error::other)?;
info!(
event = EVENT_S3_STACK_SELECTED,
component = LOG_COMPONENT_SERVER,
subsystem = LOG_SUBSYSTEM_STARTUP,
stack = "gateway",
registered_operations = stack.registered_operations().len(),
"S3 stack selected"
);
Ok(GatewayEdgeService(stack))
};
#[cfg(not(feature = "gateway-stack"))]
let assemble_gateway = || {
Err(Error::new(
std::io::ErrorKind::Unsupported,
"the gateway S3 stack is not built into this binary",
))
};
let s3_service: S3EdgeService = select_stack(config.s3_stack, s3_service, assemble_gateway)?;
if s3_service.selected() == crate::config::S3Stack::Legacy {
info!(
event = EVENT_S3_STACK_SELECTED,
component = LOG_COMPONENT_SERVER,
subsystem = LOG_SUBSYSTEM_STARTUP,
stack = "legacy",
"S3 stack selected"
);
}
#[cfg(feature = "http3")]
let http3_server = if let Some(server_config) = http3_server_config {
let h3_s3_service = s3_service.clone();
@@ -2970,7 +2889,7 @@ async fn start_http_server_inner(
struct ConnectionContext {
http_server: Arc<ConnBuilder<HttpTaskExecutor>>,
task_owner: HttpTaskExecutor,
s3_service: S3EdgeService,
s3_service: S3Service,
compression_config: HttpCompressionConfig,
is_console: bool,
/// Whether `RUSTFS_SERVER_DOMAINS` is configured (i.e. s3s virtual-hosted-style routing is active).
-5
View File
@@ -16,10 +16,6 @@ mod audit;
mod compress;
pub mod cors;
mod event;
#[cfg(feature = "gateway-stack")]
mod gateway_handlers;
#[cfg(feature = "gateway-stack")]
mod gateway_stack;
mod health;
mod http;
mod hybrid;
@@ -33,7 +29,6 @@ mod runtime;
pub(crate) mod runtime_sources;
mod service_state;
mod ssec_transport;
mod stack;
pub mod tls_material;
pub(crate) mod tls_publish;
+1 -1
View File
@@ -92,7 +92,7 @@ fn carries_ssec_headers(headers: &HeaderMap) -> bool {
/// protocol is only consulted as a second source because the trusted-proxy
/// layer has already decided whether the peer is allowed to assert it; a
/// request that arrives direct carries no such assertion.
pub(crate) fn is_secure_transport(connection_is_tls: bool, client_info: Option<&ClientInfo>) -> bool {
fn is_secure_transport(connection_is_tls: bool, client_info: Option<&ClientInfo>) -> bool {
if connection_is_tls {
return true;
}
-253
View File
@@ -1,253 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! The S3 protocol stack one server runs, selected once at startup (rustfs/backlog#2734 T2.5).
//!
//! Responsible for: [`S3StackService`], the service mounted where the legacy S3 service used to
//! be, and [`select_stack`], the only place a stack is chosen. The whole stack is one or the
//! other: no request is routed between stacks by its operation, and a request the gateway stack
//! refuses is answered by that refusal and never retried on the legacy stack.
//! NOT responsible for: assembling either stack (`server/http.rs` builds the legacy service,
//! `server/gateway_stack.rs` the gateway one), or the middleware around them.
//! Upstream: `server/http.rs`. Downstream: the legacy S3 service and the gateway stack.
//!
//! This file names neither stack's types: it is generic over two services with one response
//! type, so `server/http.rs` decides what they are and how their bodies meet.
use super::layer::is_sts_query_request;
use super::prefix::{HEALTH_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, has_path_prefix, is_admin_path};
use crate::admin::console::is_console_path;
use crate::config::S3Stack;
use http::{HeaderMap, Method, Request, Uri};
use std::future::Future;
use std::pin::Pin;
use std::task::{Context, Poll, ready};
use tower::Service;
/// The MinIO-compatible health probes share this prefix.
const MINIO_HEALTH_PREFIX: &str = "/minio/health";
/// The S3 service of one server: the legacy stack, or the gateway stack with the legacy router
/// kept for the routes that are not S3 operations.
#[derive(Clone)]
pub(crate) struct S3StackService<L, G> {
legacy: L,
gateway: Option<G>,
}
impl<L, G> S3StackService<L, G> {
/// The stack this service runs.
pub(crate) fn selected(&self) -> S3Stack {
match self.gateway {
Some(_) => S3Stack::Gateway,
None => S3Stack::Legacy,
}
}
}
/// Builds the service for the selected stack.
///
/// `assemble_gateway` runs only when the gateway stack is selected, and its failure is the
/// failure of the whole selection: the caller refuses to start.
///
/// # Errors
///
/// Whatever `assemble_gateway` refused with. The legacy stack is never substituted.
pub(crate) fn select_stack<L, G, E>(
stack: S3Stack,
legacy: L,
assemble_gateway: impl FnOnce() -> Result<G, E>,
) -> Result<S3StackService<L, G>, E> {
let gateway = match stack {
S3Stack::Legacy => None,
S3Stack::Gateway => Some(assemble_gateway()?),
};
Ok(S3StackService { legacy, gateway })
}
/// Whether the legacy router answers this request while the gateway stack is selected: the admin
/// and table-catalog APIs, the console, STS, health and profiling, none of which is an S3
/// operation. Every other request is the gateway's, whatever it answers.
fn is_legacy_only_route(method: &Method, uri: &Uri, headers: &HeaderMap) -> bool {
let path = uri.path();
is_admin_path(path)
|| is_console_path(path)
|| has_path_prefix(path, HEALTH_PREFIX)
|| has_path_prefix(path, MINIO_HEALTH_PREFIX)
|| path == PROFILE_CPU_PATH
|| path == PROFILE_MEMORY_PATH
|| is_sts_query_request(method, uri, headers)
}
impl<L, G, B, R, E> Service<Request<B>> for S3StackService<L, G>
where
L: Service<Request<B>, Response = http::Response<R>, Error = E>,
G: Service<Request<B>, Response = http::Response<R>, Error = E>,
L::Future: Send + 'static,
G::Future: Send + 'static,
{
type Response = http::Response<R>;
type Error = E;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
if let Some(gateway) = &mut self.gateway {
ready!(gateway.poll_ready(cx))?;
}
self.legacy.poll_ready(cx)
}
fn call(&mut self, request: Request<B>) -> Self::Future {
match &mut self.gateway {
// REMOVED BY T2.11: the gateway serves these routes itself once they are migrated.
Some(gateway) if !is_legacy_only_route(request.method(), request.uri(), request.headers()) => {
Box::pin(gateway.call(request))
}
_ => Box::pin(self.legacy.call(request)),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use http::{Response, StatusCode};
use std::convert::Infallible;
use std::future::{Ready, ready as ready_future};
/// Answers every request with its own name.
#[derive(Clone)]
struct Named(&'static str);
impl Service<Request<()>> for Named {
type Response = Response<&'static str>;
type Error = Infallible;
type Future = Ready<Result<Self::Response, Self::Error>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, _request: Request<()>) -> Self::Future {
ready_future(Ok(Response::new(self.0)))
}
}
fn request(method: Method, target: &str) -> Request<()> {
Request::builder()
.method(method)
.uri(target)
.body(())
.expect("static request")
}
async fn answered_by(service: &mut S3StackService<Named, Named>, request: Request<()>) -> &'static str {
let response = service.call(request).await.expect("infallible");
assert_eq!(response.status(), StatusCode::OK);
response.into_body()
}
fn gateway_stack() -> S3StackService<Named, Named> {
select_stack(S3Stack::Gateway, Named("legacy"), || Ok::<_, Infallible>(Named("gateway"))).expect("assembles")
}
#[test]
fn a_gateway_assembly_failure_fails_the_selection_instead_of_falling_back() {
let selection = select_stack::<_, Named, _>(S3Stack::Gateway, Named("legacy"), || Err("the assembly was refused"));
assert_eq!(selection.err(), Some("the assembly was refused"));
}
#[test]
fn the_legacy_selection_never_assembles_a_gateway() {
let service = select_stack::<_, Named, &str>(S3Stack::Legacy, Named("legacy"), || {
panic!("the legacy stack must not assemble the gateway")
})
.expect("legacy needs no assembly");
assert_eq!(service.selected(), S3Stack::Legacy);
}
#[tokio::test]
async fn the_legacy_stack_answers_every_request_itself() {
let mut service = select_stack(S3Stack::Legacy, Named("legacy"), || Ok::<_, Infallible>(Named("gateway")))
.expect("legacy needs no assembly");
for target in ["/", "/bucket/key", "/rustfs/admin/v3/info", "/health"] {
assert_eq!(answered_by(&mut service, request(Method::GET, target)).await, "legacy", "{target}");
}
}
#[tokio::test]
async fn the_gateway_stack_answers_every_s3_request_whatever_the_operation() {
let mut service = gateway_stack();
assert_eq!(service.selected(), S3Stack::Gateway);
for (method, target) in [
(Method::GET, "/"),
(Method::GET, "/bucket/key"),
(Method::PUT, "/bucket/key"),
(Method::DELETE, "/bucket"),
(Method::GET, "/bucket?versions"),
(Method::POST, "/bucket/key?uploads"),
// Prefixes of the legacy-only routes that are bucket names, not those routes.
(Method::GET, "/healthy/key"),
(Method::GET, "/rustfs/administrator"),
(Method::GET, "/minio/healthcheck"),
(Method::GET, "/profile/cpus"),
] {
assert_eq!(
answered_by(&mut service, request(method.clone(), target)).await,
"gateway",
"{method} {target}"
);
}
}
#[tokio::test]
async fn the_gateway_stack_leaves_the_non_s3_routes_to_the_legacy_router() {
let mut service = gateway_stack();
for target in [
"/rustfs/admin/v3/info",
"/minio/admin/v3/info",
"/iceberg/v1/config",
"/health",
"/health/ready",
"/minio/health/live",
"/profile/cpu",
"/profile/memory",
] {
assert_eq!(answered_by(&mut service, request(Method::GET, target)).await, "legacy", "{target}");
}
}
#[tokio::test]
async fn an_sts_form_post_stays_on_the_legacy_router_and_a_plain_post_does_not() {
let mut service = gateway_stack();
let sts = Request::builder()
.method(Method::POST)
.uri("/")
.header(http::header::CONTENT_TYPE, "application/x-www-form-urlencoded")
.body(())
.expect("static request");
assert!(
is_sts_query_request(sts.method(), sts.uri(), sts.headers()),
"the fixture is an STS request"
);
assert_eq!(answered_by(&mut service, sts).await, "legacy");
assert_eq!(answered_by(&mut service, request(Method::POST, "/")).await, "gateway");
}
}
+4 -4
View File
@@ -1334,11 +1334,11 @@ fn test_delete_bucket_broadcast_fences_target_membership_through_delivery() {
"a failed destructive delivery must remain visible for operator repair"
);
let usecase = include_str!("../app/bucket_usecase.rs");
let usecase = crate::guard_sources::SourceSet::read("src/app/bucket_usecase.rs", "src/app/bucket_usecase");
let delete = usecase
.split("async fn execute_delete_bucket_inner")
.nth(1)
.and_then(|rest| rest.split("pub async fn execute_head_bucket").next())
.after_unique_marker("async fn execute_delete_bucket_inner")
.split("pub async fn execute_head_bucket")
.next()
.expect("delete bucket usecase");
assert!(
delete
+7 -39
View File
@@ -41,8 +41,7 @@ const EVENT_SERVER_STARTING: &str = "server_starting";
const EVENT_ACTION_CREDENTIALS_INITIALIZED: &str = "action_credentials_initialized";
const EVENT_ACTION_CREDENTIALS_INITIALIZATION_FAILED: &str = "action_credentials_initialization_failed";
const DEFAULT_CREDENTIALS_WARNING_MESSAGE: &str = "Detected default root credentials; set RUSTFS_ACCESS_KEY and RUSTFS_SECRET_KEY to non-default values for production deployments";
#[cfg(not(feature = "gateway-stack"))]
const GATEWAY_STACK_NOT_BUILT: &str = "RUSTFS_S3_STACK=gateway needs a binary built with the `gateway-stack` feature, which no release build enables (rustfs/backlog#2734)";
const GATEWAY_STACK_NOT_ASSEMBLED: &str = "gateway stack is not assembled yet (rustfs/backlog#2734 T2.5)";
pub(crate) struct StartupListenContext {
pub(crate) readiness: Arc<GlobalReadiness>,
@@ -83,7 +82,12 @@ pub(crate) async fn init_startup_listen_context(
config: &Config,
instance_ctx: &Arc<InstanceContext>,
) -> Result<StartupListenContext> {
refuse_unbuilt_stack(config.s3_stack)?;
match config.s3_stack {
S3Stack::Legacy => {}
// REPLACED BY T2.5
// Refuse before any listener or storage exists; never fall back to legacy.
S3Stack::Gateway => return Err(Error::new(ErrorKind::Unsupported, GATEWAY_STACK_NOT_ASSEMBLED)),
}
crate::server::init_console_prefix()?;
log_sanitized_server_config(config);
let readiness = Arc::new(GlobalReadiness::new());
@@ -134,18 +138,6 @@ pub(crate) async fn init_startup_listen_context(
})
}
/// Refuses a selected S3 stack this binary cannot serve, before any listener or storage exists.
/// The selection is never replaced by another stack.
fn refuse_unbuilt_stack(stack: S3Stack) -> Result<()> {
match stack {
S3Stack::Legacy => Ok(()),
#[cfg(feature = "gateway-stack")]
S3Stack::Gateway => Ok(()),
#[cfg(not(feature = "gateway-stack"))]
S3Stack::Gateway => Err(Error::new(ErrorKind::Unsupported, GATEWAY_STACK_NOT_BUILT)),
}
}
pub(crate) async fn prepare_embedded_startup_config(
address: String,
access_key: String,
@@ -174,10 +166,6 @@ pub(crate) async fn prepare_embedded_startup_config(
config.secret_key = secret_key.clone();
config.region = Some(region.clone());
config.console_enable = false;
// The stack is selected for the whole process, so an embedded server reads the same variable
// the binary does.
config.s3_stack = S3Stack::from_env()?;
refuse_unbuilt_stack(config.s3_stack)?;
Ok(EmbeddedStartupConfig {
config,
@@ -530,26 +518,6 @@ mod tests {
));
}
#[cfg(not(feature = "gateway-stack"))]
#[test]
fn a_binary_without_the_gateway_stack_refuses_to_select_it() {
let err = super::refuse_unbuilt_stack(crate::config::S3Stack::Gateway).expect_err("the gateway stack is not built");
assert_eq!(err.kind(), std::io::ErrorKind::Unsupported);
assert!(err.to_string().contains("gateway-stack"), "{err}");
}
#[cfg(feature = "gateway-stack")]
#[test]
fn a_binary_with_the_gateway_stack_accepts_its_selection() {
super::refuse_unbuilt_stack(crate::config::S3Stack::Gateway).expect("the gateway stack is built");
}
#[test]
fn the_legacy_stack_is_always_selectable() {
super::refuse_unbuilt_stack(crate::config::S3Stack::Legacy).expect("legacy is the default stack");
}
#[tokio::test]
async fn init_startup_listen_context_rejects_console_s3_port_conflict() {
let mut config = Config::new("127.0.0.1:9001", vec!["/tmp/rustfs-data".to_string()]);
File diff suppressed because it is too large Load Diff
+5 -18
View File
@@ -20,10 +20,8 @@
//! counted and the count is carried on the next entry that is let through.
use crate::module_switches::is_audit_module_enabled;
use crate::storage::helper::{
AuditRequestView, DenialRequest, DenialScope, authorization_denial_entry, authorization_denial_entry_for_view,
spawn_background_with_context,
};
use crate::storage::access::{ReqInfo, request_context_from_req};
use crate::storage::helper::{DenialRequest, DenialScope, authorization_denial_entry, spawn_background_with_context};
use crate::storage::request_context::RequestContext;
use rustfs_audit::{AuditEntry, global::AuditLogger};
use rustfs_policy::policy::action::Action;
@@ -186,32 +184,21 @@ impl DenialAuditor {
scope: DenialScope,
now: Instant,
) {
self.audit_view_at(&AuditRequestView::legacy(req), now, || {
authorization_denial_entry(req, action, reason, scope)
});
}
/// Audits a denial for a request an owned edge decoded into `view`.
pub(crate) fn audit_owned(&self, view: &AuditRequestView<'_>, action: &Action, reason: &'static str, scope: DenialScope) {
self.audit_view_at(view, (self.clock)(), || authorization_denial_entry_for_view(view, action, reason, scope));
}
fn audit_view_at(&self, req: &AuditRequestView<'_>, now: Instant, build_entry: impl FnOnce() -> AuditEntry) {
if !(self.enabled)() {
return;
}
let anonymous = req.req_info.is_none_or(|info| info.cred.is_none());
let anonymous = req.extensions.get::<ReqInfo>().is_none_or(|info| info.cred.is_none());
let Admission::Emit { suppressed } = self.limiter.admit(anonymous, now) else {
return;
};
let mut entry = build_entry();
let mut entry = authorization_denial_entry(req, action, reason, scope);
if suppressed > 0 {
entry
.tags
.get_or_insert_with(Default::default)
.insert(SUPPRESSED_DENIALS_TAG.to_string(), Value::from(suppressed));
}
(self.emit)(req.request_context(), entry);
(self.emit)(request_context_from_req(req), entry);
}
}
+11 -25
View File
@@ -45,7 +45,6 @@ use crate::storage::validate_website_configuration;
use crate::table_catalog;
use http::StatusCode;
use metrics::{counter, histogram};
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_buckets as gateway_list_buckets;
use rustfs_io_metrics::record_s3_op;
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
use rustfs_s3_ops::S3Operation;
@@ -741,15 +740,6 @@ impl FS {
}
}
/// A ListBuckets value the s3s edge and the gateway's types disagree on. The seam converts every
/// member both sides hold, so this is a defect in the conversion, never the caller's fault.
fn list_buckets_seam_error(error: rustfs_gateway_types::compat::ConversionError) -> S3Error {
S3Error::with_message(
S3ErrorCode::InternalError,
format!("failed to convert ListBuckets through the gateway compatibility seam: {error}"),
)
}
#[async_trait::async_trait]
impl S3 for FS {
#[instrument(level = "debug", skip(self, req))]
@@ -1606,17 +1596,7 @@ impl S3 for FS {
// List buckets not associated with a bucket, give it bucket label "*" to denote "all".
record_s3_op(S3Operation::ListBuckets);
let usecase = s3_api::bucket_usecase_for(self);
// The use case speaks the envelope and the gateway's types; this edge only converts.
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
let input = gateway_list_buckets::input_from_s3s(req.input).map_err(list_buckets_seam_error)?;
let output = usecase.execute_list_buckets(envelope, input).await?;
let mut output = gateway_list_buckets::output_to_s3s(output).map_err(list_buckets_seam_error)?;
// The seam writes a listing without buckets as an absent member. This edge has always
// answered the element, empty or not, and clients index the answer by it.
if output.buckets.is_none() {
output.buckets = Some(Vec::new());
}
Ok(S3Response::new(output))
usecase.execute_list_buckets(req).await
}
async fn list_multipart_uploads(
@@ -2296,13 +2276,13 @@ impl S3 for FS {
/// Every mutating S3 operation that has no bucket notification event reaches the audit
/// stream only through `audit_only_operation`; a new such operation, or a dropped
/// wrapper, must fail here rather than silently go unaudited.
/// wrapper, must fail here rather than silently go unaudited. The wrappers are looked
/// up in this file and in `legacy_edge/`, where a split of this file moves them.
#[cfg(test)]
mod audit_coverage_tests {
use crate::guard_sources::SourceSet;
use rustfs_s3_ops::S3Operation;
const SOURCE: &str = include_str!("ecfs.rs");
fn is_read_only(op: S3Operation) -> bool {
let name = op.as_str().trim_start_matches("s3:");
["Get", "Head", "List", "Select"]
@@ -2320,9 +2300,15 @@ mod audit_coverage_tests {
expected.push(S3Operation::AbortMultipartUpload);
assert_eq!(expected.len(), 22, "an operation was added or removed: {expected:?}");
let source = SourceSet::read("src/storage/ecfs.rs", "src/storage/legacy_edge");
for op in expected {
let wrapper = format!("audit_only_operation(req, S3Operation::{op:?},");
assert!(SOURCE.contains(&wrapper), "{} is not run through audit_only_operation", op.as_str());
assert!(
source.contains(&wrapper),
"{} is not run through audit_only_operation in {}",
op.as_str(),
source.describe()
);
}
}
}
+32 -83
View File
@@ -19,7 +19,7 @@ use crate::storage::request_context::RequestContext;
use crate::storage::sse::KmsRequestAuditScope;
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use hashbrown::HashMap;
use http::{HeaderMap, StatusCode, Uri};
use http::StatusCode;
use metrics::counter;
use rustfs_audit::{
ApiDetails, AuditEntry, ObjectVersion,
@@ -38,64 +38,12 @@ use rustfs_utils::http::headers::AMZ_REQUEST_ID;
use s3s::{S3Request, S3Response, S3Result};
use serde_json::Value;
use std::future::Future;
use std::net::SocketAddr;
use tokio::runtime::{Builder, Handle};
/// Request type seen by the denial auditor, so `denial_audit` stays off the direct s3s surface.
pub(crate) type DenialRequest<T> = S3Request<T>;
use tracing::{Instrument, info_span, warn};
/// The request fields used to build an audit entry, shared by both request edges.
pub(crate) struct AuditRequestView<'a> {
headers: &'a HeaderMap,
uri: &'a Uri,
pub(crate) req_info: Option<&'a ReqInfo>,
client_info: Option<&'a ClientInfo>,
remote_addr: Option<SocketAddr>,
request_context: Option<&'a RequestContext>,
}
impl<'a> AuditRequestView<'a> {
pub(crate) fn legacy<T>(req: &'a S3Request<T>) -> Self {
Self {
headers: &req.headers,
uri: &req.uri,
req_info: req.extensions.get::<ReqInfo>(),
client_info: req.extensions.get::<ClientInfo>(),
remote_addr: req
.extensions
.get::<Option<RemoteAddr>>()
.and_then(|addr| addr.map(|addr| addr.0)),
request_context: req.extensions.get::<RequestContext>(),
}
}
/// The view of a request whose facts an owned edge already decoded.
pub(crate) fn from_facts(
headers: &'a HeaderMap,
uri: &'a Uri,
req_info: Option<&'a ReqInfo>,
client_info: Option<&'a ClientInfo>,
remote_addr: Option<SocketAddr>,
request_context: Option<&'a RequestContext>,
) -> Self {
Self {
headers,
uri,
req_info,
client_info,
remote_addr,
request_context,
}
}
pub(crate) fn request_context(&self) -> Option<RequestContext> {
self.request_context
.cloned()
.or_else(|| self.req_info.and_then(|info| info.request_context.clone()))
}
}
#[cfg(feature = "diagnostic-read-version-caller")]
use super::storage_api::ecstore_diagnostic;
use crate::storage::storage_api::helper_consumer::StorageObjectInfo as ObjectInfo;
@@ -179,27 +127,32 @@ pub(crate) fn build_event_resp_elements<T>(response: &S3Response<T>, request_id:
/// Builds the audit entry shared by every request-scoped audit source: caller identity,
/// client address and the request line, without any outcome.
fn request_audit_builder(
req: &AuditRequestView<'_>,
fn request_audit_builder<T>(
req: &S3Request<T>,
req_info: Option<&ReqInfo>,
event: EventName,
trigger: &str,
api: ApiDetails,
request_id: &str,
) -> AuditEntryBuilder {
let remote_host = req
.client_info
.extensions
.get::<ClientInfo>()
.map(|info| info.real_ip.to_string())
.or_else(|| req.remote_addr.map(|addr| addr.ip().to_string()))
.or_else(|| {
req.extensions
.get::<Option<RemoteAddr>>()
.and_then(|addr| addr.map(|addr| addr.0.ip().to_string()))
})
.unwrap_or_default();
let mut builder = AuditEntryBuilder::new("1.0", event, trigger, api)
.remote_host(remote_host)
.user_agent(get_request_user_agent(req.headers))
.req_host(get_request_host(req.headers))
.user_agent(get_request_user_agent(&req.headers))
.req_host(get_request_host(&req.headers))
.req_path(req.uri.path().to_string())
.req_query(extract_params_header(req.headers))
.req_query(extract_params_header(&req.headers))
.request_id(request_id);
if let Some(cred) = req
.req_info
if let Some(cred) = req_info
.and_then(|info| info.cred.as_ref())
.filter(|cred| !cred.access_key.is_empty())
{
@@ -421,9 +374,8 @@ impl OperationHelper {
let request_context = request_context.unwrap_or_else(|| RequestContext::from_external_headers(&req.headers));
let request_id = request_context.request_id.clone();
let audit_builder = audit_enabled.then(|| {
request_audit_builder(&AuditRequestView::legacy(req), event, trigger, api_builder.clone().build(), &request_id)
});
let audit_builder =
audit_enabled.then(|| request_audit_builder(req, req_info, event, trigger, api_builder.clone().build(), &request_id));
let event_object = ObjectInfo {
bucket: bucket.clone(),
@@ -650,16 +602,7 @@ pub(crate) fn authorization_denial_entry<T>(
reason: &'static str,
scope: DenialScope,
) -> AuditEntry {
authorization_denial_entry_for_view(&AuditRequestView::legacy(req), action, reason, scope)
}
pub(crate) fn authorization_denial_entry_for_view(
req: &AuditRequestView<'_>,
action: &Action,
reason: &'static str,
scope: DenialScope,
) -> AuditEntry {
let req_info = req.req_info;
let req_info = req.extensions.get::<ReqInfo>();
let path = req.uri.path().trim_start_matches('/');
let mut segs = path.splitn(2, '/');
let path_bucket = segs.next().unwrap_or("");
@@ -688,18 +631,23 @@ pub(crate) fn authorization_denial_entry_for_view(
api = api.object(object);
}
let request_context = req
.request_context()
.unwrap_or_else(|| RequestContext::from_external_headers(req.headers));
let request_context = request_context_from_req(req).unwrap_or_else(|| RequestContext::from_external_headers(&req.headers));
let mut tags = HashMap::new();
tags.insert("denialReason".to_string(), Value::String(reason.to_string()));
if let DenialScope::Keys(denied) = scope {
tags.insert("deniedKeys".to_string(), Value::from(denied));
}
request_audit_builder(req, EventName::AuditS3Operation, action_name, api.build(), &request_context.request_id)
.error("Access Denied")
.tags(tags)
.build()
request_audit_builder(
req,
req_info,
EventName::AuditS3Operation,
action_name,
api.build(),
&request_context.request_id,
)
.error("Access Denied")
.tags(tags)
.build()
}
/// Runs `handler` on `req` and records the outcome as an audit-only entry for `op`.
@@ -1487,7 +1435,8 @@ mod tests {
..Default::default()
});
let entry = super::request_audit_builder(
&super::AuditRequestView::legacy(&req),
&req,
req.extensions.get::<ReqInfo>(),
EventName::ObjectAccessedGet,
"GetObject",
rustfs_audit::ApiDetails::default(),
+58 -7
View File
@@ -13,14 +13,15 @@
// limitations under the License.
use crate::storage::s3_api::common::rustfs_owner;
use crate::storage::storage_api::s3_api_consumer::bucket::contract::list::{
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
use crate::storage::storage_api::s3_api_consumer::bucket::contract::{
bucket::BucketInfo,
list::{ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info},
};
use crate::storage::storage_api::s3_api_consumer::bucket::to_s3s_xml_etag;
use percent_encoding::percent_decode_str;
use s3s::dto::{
CommonPrefix, DeleteMarkerEntry, EncodingType, ListObjectVersionsOutput, ListObjectsOutput, ListObjectsV2Output, Object,
ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Timestamp,
Bucket, CommonPrefix, DeleteMarkerEntry, EncodingType, ListBucketsOutput, ListObjectVersionsOutput, ListObjectsOutput,
ListObjectsV2Output, Object, ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Timestamp,
};
use s3s::{S3Error, S3ErrorCode};
use tracing::debug;
@@ -125,6 +126,23 @@ pub(crate) fn build_list_objects_output(v2: ListObjectsV2Output, request_marker:
}
}
pub(crate) fn build_list_buckets_output(bucket_infos: &[BucketInfo]) -> ListBucketsOutput {
let buckets: Vec<Bucket> = bucket_infos
.iter()
.map(|bucket_info| Bucket {
creation_date: Some(Timestamp::from(bucket_info.created.unwrap_or(time::OffsetDateTime::UNIX_EPOCH))),
name: Some(bucket_info.name.clone()),
..Default::default()
})
.collect();
ListBucketsOutput {
buckets: Some(buckets),
owner: Some(rustfs_owner()),
..Default::default()
}
}
pub(crate) fn parse_list_object_versions_params(
prefix: Option<String>,
delimiter: Option<String>,
@@ -432,16 +450,19 @@ fn calculate_next_marker(v2: &ListObjectsV2Output) -> Option<String> {
#[cfg(test)]
mod tests {
use super::{
ListObjectVersionsInfo, ListObjectVersionsParams, ListObjectsV2Info, build_list_object_versions_output,
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
parse_list_object_versions_params, parse_list_objects_v2_params, to_s3s_xml_etag,
ListObjectVersionsInfo, ListObjectVersionsParams, ListObjectsV2Info, build_list_buckets_output,
build_list_object_versions_output, build_list_objects_output, build_list_objects_v2_output,
encode_list_objects_v2_continuation_token, parse_list_object_versions_params, parse_list_objects_v2_params,
to_s3s_xml_etag,
};
use crate::storage::s3_api::common::rustfs_owner;
use crate::storage::storage_api::s3_api_consumer::bucket::StorageObjectInfo as ObjectInfo;
use crate::storage::storage_api::s3_api_consumer::bucket::contract::bucket::BucketInfo;
use rustfs_filemeta::FileInfo;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
use s3s::S3ErrorCode;
use s3s::dto::{CommonPrefix, EncodingType, ListObjectsV2Output, Object};
use time::OffsetDateTime;
use uuid::Uuid;
#[test]
@@ -504,6 +525,36 @@ mod tests {
}
}
#[test]
fn test_build_list_buckets_output_maps_bucket_infos_and_owner() {
let bucket_infos = vec![
BucketInfo {
name: "bucket-a".to_string(),
created: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
},
BucketInfo {
name: "bucket-b".to_string(),
created: None,
..Default::default()
},
];
let output = build_list_buckets_output(&bucket_infos);
let buckets = output.buckets.as_ref().expect("buckets should be present");
let owner = output.owner.as_ref().expect("owner should be present");
assert_eq!(buckets.len(), 2);
assert_eq!(buckets[0].name.as_deref(), Some("bucket-a"));
assert_eq!(buckets[0].creation_date, Some(s3s::dto::Timestamp::from(OffsetDateTime::UNIX_EPOCH)));
assert_eq!(buckets[1].name.as_deref(), Some("bucket-b"));
assert_eq!(buckets[1].creation_date, Some(s3s::dto::Timestamp::from(OffsetDateTime::UNIX_EPOCH)));
let expected_owner = rustfs_owner();
assert_eq!(owner.display_name, expected_owner.display_name);
assert_eq!(owner.id, expected_owner.id);
}
#[test]
fn test_list_objects_marker_echoes_request_value() {
let output = build_list_objects_output(ListObjectsV2Output::default(), Some("m-1".to_string()));
+2 -5
View File
@@ -25,14 +25,11 @@ impl RequestEnvelope {
/// Copies every request fact of `req` the app layer reads.
pub fn from_s3s<T>(req: &S3Request<T>) -> Self {
let credentials = req.credentials.as_ref().map(|credentials| {
let session_token = get_session_token(&req.uri, &req.headers).map(str::to_owned);
// An empty secret is not a key: carry it as absent so every reader fails closed.
RequestCredentials::new(
credentials.access_key.clone(),
credentials.secret_key.expose().to_owned(),
session_token.clone(),
get_session_token(&req.uri, &req.headers).map(str::to_owned),
)
.unwrap_or_else(|_| RequestCredentials::without_secret_key(credentials.access_key.clone(), session_token))
});
let head = RequestHead {
method: req.method.clone(),
@@ -181,7 +178,7 @@ mod tests {
assert_eq!(envelope.headers(), &req.headers);
let credentials = envelope.credentials().expect("credentials");
assert_eq!(credentials.access_key(), "AKIAENVELOPE");
assert_eq!(credentials.secret_key(), Some("envelope-secret"));
assert_eq!(credentials.secret_key(), "envelope-secret");
assert_eq!(credentials.session_token(), Some("header-token"));
assert_eq!(envelope.region(), Some("eu-west-3"));
assert_eq!(envelope.service(), Some("s3"));
+17 -32
View File
@@ -147,36 +147,17 @@ pub(crate) use super::sse::{
pub(crate) mod access_consumer {
pub(crate) use super::super::access::{
AuthorizationRequest, BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard,
ListBucketsAuthzRequest, MultipartStagingOperation, ObjectTagConditions, OdmReadGenerationGuard,
PendingDeleteBucketGenerationGuard, PostObjectPolicyHeaders, PostObjectRequestMarker, ReqInfo, RequestedCopySource,
RequestedObjectLegalHold, RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags,
StagedMultipartPartAuthorization, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess,
TableDataPlaneListCursorPosition, TableDataPlanePublicationGuards, apply_bucket_generation_guard,
apply_copy_source_bucket_generation_guard, authorize_copy_object_tags, authorize_request, authorize_request_core,
bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header,
load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller,
replication_proxy_allowed, replication_request_authorized, req_info_mut, req_info_ref,
retained_table_data_plane_publication_guards, set_requested_object_lock_retain_until,
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, MultipartStagingOperation,
ObjectTagConditions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectRequestMarker, ReqInfo,
TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, TableDataPlaneListCursorPosition,
TableDataPlanePublicationGuards, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard,
authorize_copy_object_tags, authorize_internal_object_request, authorize_request, bucket_config_mutation_incarnation,
delete_object_authorize_action, has_bypass_governance_header, load_bucket_generation_from_store,
log_list_buckets_iam_implicit_deny, odm_read_generation, prepare_list_buckets_iam_authorization,
prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller, replication_proxy_allowed,
replication_request_authorized, req_info_mut, req_info_ref, retained_table_data_plane_publication_guards,
set_requested_object_lock_retain_until,
};
#[cfg(feature = "gateway-stack")]
pub(crate) use super::super::access::{admit_operation, resolve_req_info};
pub(crate) use super::super::denial_audit::DenialAuditor;
pub(crate) use super::super::helper::{AuditRequestView, DenialScope};
}
#[cfg(test)]
pub(crate) mod owned_parity_consumer {
pub(crate) use super::super::access::{
ObjectTagConditions, PostObjectPolicyHeaders, PostObjectRequestMarker, ReqInfo, RequestedCopySource,
RequestedObjectLegalHold, RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags,
TableDataPlanePublicationGuards, authorize_request, parity_test_support,
};
pub(crate) use super::super::denial_audit::{DenialAuditor, DenialRateLimiter, SUPPRESSED_DENIALS_TAG};
pub(crate) use super::super::ecfs::FS;
pub(crate) use super::super::helper::{DenialScope, authorization_denial_entry, authorization_denial_entry_for_view};
pub(crate) use super::super::request_context::RequestContext;
}
pub(crate) mod concurrency_consumer {
@@ -371,12 +352,16 @@ pub(crate) mod heal_control_startup_consumer {
pub(crate) mod s3_api_consumer {
pub(crate) mod bucket {
pub(crate) use super::super::super::s3_api::bucket::{
ListObjectVersionsParams, ListObjectsV2Params, build_list_object_versions_output, build_list_objects_output,
build_list_objects_v2_output, encode_list_objects_v2_continuation_token, parse_list_object_versions_params,
parse_list_objects_v2_params,
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
parse_list_object_versions_params, parse_list_objects_v2_params,
};
pub(crate) mod contract {
pub(crate) mod bucket {
pub(crate) use super::super::super::super::contract::bucket::BucketInfo;
}
pub(crate) mod list {
pub(crate) use super::super::super::super::contract::list::{ListObjectVersionsInfo, ListObjectsV2Info};
}
-7
View File
@@ -231,13 +231,6 @@ pub(crate) mod server {
pub(crate) use crate::storage::storage_api::request_context_consumer::RequestContext;
}
/// The access-phase functions the gateway stack's handlers run, shared with the legacy
/// access hooks.
#[cfg(feature = "gateway-stack")]
pub(crate) mod access {
pub(crate) use crate::storage::storage_api::access_consumer::{admit_operation, resolve_req_info};
}
pub(crate) mod rpc {
pub(crate) use crate::storage::storage_api::rpc_consumer::InternodeRpcService;
#[cfg(test)]
+14 -14
View File
@@ -14478,31 +14478,31 @@ fn object_mutation_entrypoints_call_reserved_prefix_guard() {
#[test]
fn multipart_mutation_entrypoints_call_reserved_prefix_guard() {
let source = include_str!("../app/multipart_usecase.rs");
let source = crate::guard_sources::SourceSet::read("src/app/multipart_usecase.rs", "src/app/multipart_usecase");
assert_eq!(
source
.matches("validate_table_catalog_object_mutation(&bucket, &key).await?;")
.count(),
4
source.count("validate_table_catalog_object_mutation(&bucket, &key).await?;"),
4,
"reserved-prefix guard calls across {}",
source.describe()
);
}
#[test]
fn object_metadata_mutation_entrypoints_call_reserved_prefix_guard() {
let source = include_str!("../storage/ecfs.rs");
let source = crate::guard_sources::SourceSet::read("src/storage/ecfs.rs", "src/storage/legacy_edge");
assert_eq!(
source
.matches("validate_table_catalog_object_mutation(&bucket, &object).await?;")
.count(),
2
source.count("validate_table_catalog_object_mutation(&bucket, &object).await?;"),
2,
"reserved-prefix guard calls across {}",
source.describe()
);
assert_eq!(
source
.matches("validate_table_catalog_object_mutation(&bucket, &key).await?;")
.count(),
2
source.count("validate_table_catalog_object_mutation(&bucket, &key).await?;"),
2,
"reserved-prefix guard calls across {}",
source.describe()
);
}
-402
View File
@@ -1,402 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The gateway S3 stack inside an embedded RustFS server (rustfs/backlog#2734 T2.5): one
// operation, ListBuckets, is served end to end, and every other S3 operation is answered
// `501 NotImplemented`. Requests are signed by the AWS SDK and travel through the whole
// middleware stack, so what is asserted here is what a client sees.
#![recursion_limit = "256"]
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::{Client, Config};
use rustfs::app::context::get_global_app_context;
use rustfs::embedded::{RustFSServer, RustFSServerBuilder, find_available_port};
use rustfs_io_metrics::s3_http_metrics::s3_http_metrics_snapshot;
use rustfs_s3_ops::S3Operation;
use rustfs_storage_api::{BucketOperations, MakeBucketOptions};
use std::collections::BTreeMap;
use std::future::Future;
mod common;
const ROOT_ACCESS_KEY: &str = "gatewaystackroot";
const ROOT_SECRET_KEY: &str = "gateway-stack-root-secret";
const USER_SECRET_KEY: &str = "gateway-stack-user-secret";
const ENV_S3_STACK: &str = "RUSTFS_S3_STACK";
const ENV_KEYSTONE_ENABLE: &str = "RUSTFS_KEYSTONE_ENABLE";
const ENV_KEYSTONE_AUTH_URL: &str = "RUSTFS_KEYSTONE_AUTH_URL";
/// A Keystone endpoint nothing listens on: these tests are about start-up, not about tokens.
const UNREACHABLE_KEYSTONE: &str = "http://127.0.0.1:1/v3";
fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client {
let config = Config::builder()
.credentials_provider(Credentials::new(access_key, secret_key, None, None, "test"))
.region(Region::new("us-east-1"))
.endpoint_url(endpoint)
.force_path_style(true)
.behavior_version_latest()
.build();
Client::from_conf(config)
}
/// Runs `test` against an embedded server started with `RUSTFS_S3_STACK` set to `stack`.
fn with_stack<F, Fut>(stack: Option<&'static str>, test: F)
where
F: FnOnce(RustFSServer) -> Fut + Send + 'static,
Fut: Future<Output = ()> + 'static,
{
with_environment(vec![(ENV_S3_STACK, stack)], test);
}
/// Runs `test` against an embedded server started under `environment`.
fn with_environment<F, Fut>(environment: Vec<(&'static str, Option<&'static str>)>, test: F)
where
F: FnOnce(RustFSServer) -> Fut + Send + 'static,
Fut: Future<Output = ()> + 'static,
{
temp_env::with_vars(environment, || {
common::run_embedded_test(move || async move {
let Some(started) = start_server().await else { return };
test(started.expect("start embedded server")).await;
});
});
}
/// Starts an embedded server on a free port; `None` when the sandbox allows no port at all.
async fn start_server() -> Option<Result<RustFSServer, String>> {
let port = match find_available_port() {
Ok(port) => port,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
Err(err) => panic!("find free port: {err}"),
};
let started = RustFSServerBuilder::new()
.address(format!("127.0.0.1:{port}"))
.access_key(ROOT_ACCESS_KEY)
.secret_key(ROOT_SECRET_KEY)
.build()
.await;
// Both renderings: the refusal's text may be in either.
Some(started.map_err(|error| format!("{error} {error:?}")))
}
/// The S3 error code and HTTP status an SDK call was refused with.
fn refusal<E: ProvideErrorMetadata>(error: &SdkError<E>) -> (Option<String>, Option<u16>) {
let status = error.raw_response().map(|response| response.status().as_u16());
(error.code().map(str::to_owned), status)
}
/// One series of the S3 request counter: method, operation and outcome.
type Series = (&'static str, &'static str, &'static str);
/// How often each series of the S3 request counter has been counted in this process.
fn s3_request_counts() -> BTreeMap<Series, u64> {
s3_http_metrics_snapshot()
.into_iter()
.map(|series| ((series.method, series.operation, series.outcome), series.total))
.collect()
}
/// Every series counted since `before`, with how often. Reading the whole counter, not one
/// series, is what makes a second count under another label visible.
fn counted_since(before: &BTreeMap<Series, u64>) -> BTreeMap<Series, u64> {
s3_request_counts()
.into_iter()
.filter_map(|(series, total)| {
let counted = total - before.get(&series).copied().unwrap_or(0);
(counted > 0).then_some((series, counted))
})
.collect()
}
async fn create_bucket_in_the_store(bucket: &str) {
get_global_app_context()
.expect("embedded server context")
.object_store()
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket in the store");
}
async fn create_user_without_a_policy(access_key: &str) {
get_global_app_context()
.expect("embedded server context")
.iam()
.handle()
.create_user(
access_key,
&rustfs_madmin::AddOrUpdateUserReq {
secret_key: USER_SECRET_KEY.to_string(),
policy: None,
status: rustfs_madmin::AccountStatus::Enabled,
},
)
.await
.expect("create user without a policy");
}
#[test]
fn the_gateway_stack_lists_the_caller_s_buckets_and_answers_every_other_operation_501() {
with_stack(Some("gateway"), |server| async move {
// CreateBucket is not served by this stack yet, so the buckets are made in the store.
create_bucket_in_the_store("gateway-stack-alpha").await;
create_bucket_in_the_store("gateway-stack-beta").await;
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
let listing = client.list_buckets().send().await.expect("signed ListBuckets");
let mut names: Vec<&str> = listing.buckets().iter().filter_map(|bucket| bucket.name()).collect();
names.sort_unstable();
assert_eq!(names, ["gateway-stack-alpha", "gateway-stack-beta"]);
assert!(listing.owner().and_then(|owner| owner.id()).is_some(), "the listing names its owner");
assert!(
listing.buckets().iter().all(|bucket| bucket.creation_date().is_some()),
"every bucket carries its creation date"
);
let get = client
.get_object()
.bucket("gateway-stack-alpha")
.key("object")
.send()
.await
.expect_err("GetObject is not registered");
assert_eq!(refusal(&get), (Some("NotImplemented".to_owned()), Some(501)));
let put = client
.put_object()
.bucket("gateway-stack-alpha")
.key("object")
.body(ByteStream::from_static(b"payload"))
.send()
.await
.expect_err("PutObject is not registered");
assert_eq!(refusal(&put), (Some("NotImplemented".to_owned()), Some(501)));
let create = client
.create_bucket()
.bucket("gateway-stack-gamma")
.send()
.await
.expect_err("CreateBucket is not registered");
assert_eq!(refusal(&create), (Some("NotImplemented".to_owned()), Some(501)));
let listing = client.list_buckets().send().await.expect("ListBuckets after the refusals");
assert_eq!(listing.buckets().len(), 2, "a refused write created nothing");
server.shutdown().await;
});
}
#[test]
fn the_gateway_stack_refuses_callers_it_cannot_admit() {
with_stack(Some("gateway"), |server| async move {
create_bucket_in_the_store("gateway-stack-alpha").await;
create_user_without_a_policy("gatewaystacknopolicy").await;
let endpoint = server.endpoint();
// Anonymous: no credentials at all.
let anonymous = reqwest::Client::builder()
.no_proxy()
.build()
.expect("http client")
.get(format!("{endpoint}/"))
.send()
.await
.expect("anonymous ListBuckets is answered");
assert_eq!(anonymous.status().as_u16(), 403);
let request_id = anonymous
.headers()
.get("x-amz-request-id")
.and_then(|value| value.to_str().ok())
.map(str::to_owned)
.expect("the refusal names its request");
assert_eq!(
anonymous.headers().get("x-request-id").and_then(|value| value.to_str().ok()),
Some(request_id.as_str()),
"one identifier names the request in both headers"
);
let body = anonymous.text().await.expect("refusal body");
assert!(body.contains("<Code>AccessDenied</Code>"), "{body}");
// The right access key with the wrong secret.
let wrong_secret = s3_client(&endpoint, ROOT_ACCESS_KEY, "not-the-root-secret")
.list_buckets()
.send()
.await
.expect_err("a wrong secret signs nothing");
assert_eq!(refusal(&wrong_secret), (Some("SignatureDoesNotMatch".to_owned()), Some(403)));
// An access key nobody issued: the caller's fault, not the server's.
let unknown_key = s3_client(&endpoint, "gatewaystackunknown", USER_SECRET_KEY)
.list_buckets()
.send()
.await
.expect_err("an unknown access key is refused");
assert_eq!(refusal(&unknown_key), (Some("InvalidAccessKeyId".to_owned()), Some(403)));
// A real user whose policy grants nothing: it may neither list every bucket nor see one.
let no_permission = s3_client(&endpoint, "gatewaystacknopolicy", USER_SECRET_KEY)
.list_buckets()
.send()
.await
.expect_err("a caller without permission lists nothing");
assert_eq!(refusal(&no_permission), (Some("AccessDenied".to_owned()), Some(403)));
// The root caller is still served by the same server.
let listing = s3_client(&endpoint, server.access_key(), server.secret_key())
.list_buckets()
.send()
.await
.expect("signed ListBuckets");
assert_eq!(listing.buckets().len(), 1);
server.shutdown().await;
});
}
/// The S3 request counter belongs to RustFS's layer in front of both stacks, and the operation
/// label is set by the code a handler calls. That only works if the gateway polls its handler
/// inside that layer's scope, on the request's own task, which is what this observes.
#[test]
fn the_gateway_stack_counts_each_request_once_under_the_label_the_legacy_stack_uses() {
with_stack(Some("gateway"), |server| async move {
let endpoint = server.endpoint();
let before = s3_request_counts();
s3_client(&endpoint, server.access_key(), server.secret_key())
.list_buckets()
.send()
.await
.expect("signed ListBuckets");
assert_eq!(
counted_since(&before),
BTreeMap::from([(("GET", S3Operation::ListBuckets.as_str(), "2xx"), 1)]),
"one served request is counted once, as ListBuckets"
);
let before = s3_request_counts();
let refused = s3_client(&endpoint, ROOT_ACCESS_KEY, "not-the-root-secret")
.list_buckets()
.send()
.await
.expect_err("a wrong secret signs nothing");
assert_eq!(refusal(&refused), (Some("SignatureDoesNotMatch".to_owned()), Some(403)));
assert_eq!(
counted_since(&before),
BTreeMap::from([(("GET", "unknown", "4xx"), 1)]),
"a request refused by authentication is counted once, under no operation"
);
server.shutdown().await;
});
}
/// The pinned gateway admits a request that carries only `X-Auth-Token` as anonymous, while the
/// RustFS code behind it reads the Keystone identity. Until the gateway authenticates the token
/// (rustfs/gateway#1586) the two settings together refuse to start, and nothing is served.
#[test]
fn the_gateway_stack_refuses_to_start_with_keystone_enabled() {
let environment = [
(ENV_S3_STACK, Some("gateway")),
(ENV_KEYSTONE_ENABLE, Some("true")),
(ENV_KEYSTONE_AUTH_URL, Some(UNREACHABLE_KEYSTONE)),
];
temp_env::with_vars(environment, || {
common::run_embedded_test(|| async {
let Some(started) = start_server().await else { return };
let refusal = match started {
Ok(server) => {
server.shutdown().await;
panic!("the gateway stack started with Keystone enabled");
}
Err(refusal) => refusal,
};
for named in ["RUSTFS_S3_STACK=gateway", "RUSTFS_KEYSTONE_ENABLE", "rustfs/gateway#1586"] {
assert!(refusal.contains(named), "{named} is missing from: {refusal}");
}
});
});
}
#[test]
fn the_legacy_stack_starts_with_keystone_enabled() {
let environment = vec![
(ENV_S3_STACK, None),
(ENV_KEYSTONE_ENABLE, Some("true")),
(ENV_KEYSTONE_AUTH_URL, Some(UNREACHABLE_KEYSTONE)),
];
with_environment(environment, |server| async move {
let listing = s3_client(&server.endpoint(), server.access_key(), server.secret_key())
.list_buckets()
.send()
.await
.expect("signed ListBuckets on the legacy stack with Keystone enabled");
assert!(listing.buckets().is_empty());
server.shutdown().await;
});
}
#[test]
fn the_legacy_stack_still_serves_every_operation() {
with_stack(None, |server| async move {
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
client
.create_bucket()
.bucket("legacy-stack-alpha")
.send()
.await
.expect("CreateBucket on the legacy stack");
client
.put_object()
.bucket("legacy-stack-alpha")
.key("object")
.body(ByteStream::from_static(b"payload"))
.send()
.await
.expect("PutObject on the legacy stack");
let before = s3_request_counts();
let listing = client.list_buckets().send().await.expect("ListBuckets on the legacy stack");
assert_eq!(
counted_since(&before),
BTreeMap::from([(("GET", S3Operation::ListBuckets.as_str(), "2xx"), 1)]),
"the label the gateway stack is held to is the one this stack counts under"
);
let names: Vec<&str> = listing.buckets().iter().filter_map(|bucket| bucket.name()).collect();
assert_eq!(names, ["legacy-stack-alpha"]);
assert!(listing.owner().and_then(|owner| owner.id()).is_some(), "the listing names its owner");
let missing = client
.get_object()
.bucket("legacy-stack-alpha")
.key("missing")
.send()
.await
.expect_err("a missing key");
assert_eq!(refusal(&missing), (Some("NoSuchKey".to_owned()), Some(404)));
create_user_without_a_policy("legacystacknopolicy").await;
let no_permission = s3_client(&server.endpoint(), "legacystacknopolicy", USER_SECRET_KEY)
.list_buckets()
.send()
.await
.expect_err("a caller without permission lists nothing on either stack");
assert_eq!(refusal(&no_permission), (Some("AccessDenied".to_owned()), Some(403)));
server.shutdown().await;
});
}
+1
View File
@@ -38,6 +38,7 @@ their issue closes.
| `check_no_tokio_io_uring.sh` | ci-gate | Keeps tokio's io-uring backend disabled | ci.yml Quick Checks |
| `check_s3s_footprint.sh` | ci-gate | Exact, lower-only baseline on the direct s3s surface ahead of the gateway migration (reads `.config/s3s-footprint-baseline.txt`; `s3_error_lines` leaves out lines whose file imports the macro from `rustfs-s3-types`, rule in the script header); `--mode allowlist` fails on any Rust file referencing s3s outside `.config/s3s-edge-allowlist.txt`, `--dry-run` lists them | ci.yml Quick Checks; `make pre-commit`; `make script-tests` (`--self-test`) |
| `check_test_count_ratchet.sh` | ci-gate | Per-crate test-count floor and `#[ignore]` ceiling against `.config/test-count-baseline.txt`; no crate may fall below its tests row or rise above its ignored row (extra tests and new crates pass with a note) | ci.yml Quick Checks; `make pre-commit` |
| `guard_source_set.sh` | ci-gate (library) | Sourced helper that resolves a text guard's input as a required file plus the directory a mechanical split moves its code into (rustfs/backlog#2765); a missing required file fails | Sourced by `check_architecture_migration_rules.sh` and `check_logging_guardrails.sh`; contract tests `test_guard_source_set.sh` (`make script-tests`) |
| `check_unsafe_code_allowances.sh` | ci-gate | Unsafe-code allowance ledger guard | ci.yml Quick Checks |
| `layer-dependency-baseline.txt` | ci-gate (data) | Committed baseline consumed by `check_layer_dependencies.sh` | [Architecture guard troubleshooting](../docs/operations/architecture-guard-troubleshooting.md) |
| `static.sh` | ci-gate | Static-build helper executed inside image builds | `Dockerfile.source`, `Dockerfile.decommission-local` |
+38 -9
View File
@@ -118,6 +118,28 @@ require_source_contains "docs/architecture/ecstore-api-facade-inventory.md" "Do
TMP_DIR="$(mktemp -d)"
trap 'rm -rf "$TMP_DIR"' EXIT
# Rules about one of the large files below read that file plus the sibling
# directory a mechanical split moves its code into (rustfs/backlog#2765), so a
# split neither breaks nor silently blinds them. The *_SPLIT_SOURCES arrays hold
# only the directory part, for rules that select the file itself by glob.
# shellcheck source=scripts/guard_source_set.sh
source "${ROOT_DIR}/scripts/guard_source_set.sh"
guard_source_set "$ROOT_DIR" rustfs/src/storage/ecfs.rs rustfs/src/storage/legacy_edge ||
report_failure "ECFS guard source set is unreadable"
ECFS_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
guard_source_set "$ROOT_DIR" rustfs/src/app/bucket_usecase.rs rustfs/src/app/bucket_usecase ||
report_failure "bucket usecase guard source set is unreadable"
BUCKET_USECASE_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
BUCKET_USECASE_SPLIT_SOURCES=("${GUARD_SOURCES[@]:1}")
guard_source_set "$ROOT_DIR" rustfs/src/app/multipart_usecase.rs rustfs/src/app/multipart_usecase ||
report_failure "multipart usecase guard source set is unreadable"
MULTIPART_USECASE_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
MULTIPART_USECASE_SPLIT_SOURCES=("${GUARD_SOURCES[@]:1}")
guard_source_set "$ROOT_DIR" rustfs/src/app/lifecycle_transition_api_test.rs rustfs/src/app/lifecycle_transition_api_test ||
report_failure "lifecycle transition API test guard source set is unreadable"
LIFECYCLE_TRANSITION_TEST_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES=("${GUARD_SOURCES[@]:1}")
PR_TYPES_FILE="${TMP_DIR}/pr_types.txt"
PR_TYPE_HITS_FILE="${TMP_DIR}/pr_type_hits.txt"
SOURCE_MARKERS_FILE="${TMP_DIR}/source_markers.txt"
@@ -2157,7 +2179,7 @@ fi
cd "$ROOT_DIR"
rg -n --with-filename 'crate::app::context::(?:\{[^}]*\b(?:resolve_object_store_handle|resolve_buffer_config|resolve_internode_metrics|resolve_local_node_name|resolve_action_credentials|resolve_notify_interface|resolve_performance_metrics|resolve_encryption_service|resolve_region|resolve_ready_iam_handle)\b|(?:resolve_object_store_handle|resolve_buffer_config|resolve_internode_metrics|resolve_local_node_name|resolve_action_credentials|resolve_notify_interface|resolve_performance_metrics|resolve_encryption_service|resolve_region|resolve_ready_iam_handle)\b)' \
rustfs/src/storage/access.rs \
rustfs/src/storage/ecfs.rs \
"${ECFS_GUARD_SOURCES[@]}" \
rustfs/src/storage/ecfs_extend.rs \
rustfs/src/storage/helper.rs \
rustfs/src/storage/concurrency/manager.rs \
@@ -2175,7 +2197,7 @@ fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'crate::app::(?:bucket_usecase|multipart_usecase|object_usecase|object\b)|Default(?:Bucket|Multipart|Object)Usecase::from_global\(\)' \
rustfs/src/storage/ecfs.rs \
"${ECFS_GUARD_SOURCES[@]}" \
--glob '*.rs' || true
) >"$RUSTFS_STORAGE_ECFS_USECASE_BYPASS_HITS_FILE"
@@ -2228,7 +2250,10 @@ fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'use crate::storage::\*;' rustfs/src/app --glob '*_usecase.rs' --glob 'object/*.rs' || true
# A glob containing a slash is anchored at the search root, so object files
# need the leading **/ to match under rustfs/src/app.
rg -n --with-filename 'use crate::storage::\*;' rustfs/src/app --glob '*_usecase.rs' --glob '**/object/*.rs' \
"${BUCKET_USECASE_SPLIT_SOURCES[@]}" "${MULTIPART_USECASE_SPLIT_SOURCES[@]}" || true
) >"$RUSTFS_APP_USECASE_STORAGE_WILDCARD_HITS_FILE"
if [[ -s "$RUSTFS_APP_USECASE_STORAGE_WILDCARD_HITS_FILE" ]]; then
@@ -2246,7 +2271,8 @@ fi
(
cd "$ROOT_DIR"
rg -n --with-filename 'crate::storage::s3_api::|use crate::storage::s3_api|super::s3_api::|use super::s3_api' rustfs/src/app --glob '*_usecase.rs' --glob 'object/*.rs' || true
rg -n --with-filename 'crate::storage::s3_api::|use crate::storage::s3_api|super::s3_api::|use super::s3_api' rustfs/src/app --glob '*_usecase.rs' --glob '**/object/*.rs' \
"${BUCKET_USECASE_SPLIT_SOURCES[@]}" "${MULTIPART_USECASE_SPLIT_SOURCES[@]}" || true
) >"$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE"
if [[ -s "$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE" ]]; then
@@ -2258,13 +2284,14 @@ fi
{
rg -n --with-filename \
'(use crate::storage::(access|helper|options|request_context|sse|timeout_wrapper|head_prefix|concurrency|ecfs)|crate::storage::sse::EncryptionKeyKind|use crate::storage::\{|use crate::storage::[A-Z])' \
rustfs/src/app/select_object.rs rustfs/src/app/*_usecase.rs rustfs/src/app/object || true
rustfs/src/app/select_object.rs rustfs/src/app/*_usecase.rs rustfs/src/app/object \
"${BUCKET_USECASE_SPLIT_SOURCES[@]}" "${MULTIPART_USECASE_SPLIT_SOURCES[@]}" || true
rg -n --with-filename \
'use super::(?:\{[^}]*\b(?:DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|StorageError|DiskError|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b|(?:object_api_utils::to_s3s_etag|storageclass|StorageError|DiskError|DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b)' \
rustfs/src/app/bucket_usecase.rs rustfs/src/app/object_usecase.rs rustfs/src/app/object rustfs/src/app/multipart_usecase.rs rustfs/src/app/lifecycle_transition_api_test.rs || true
"${BUCKET_USECASE_GUARD_SOURCES[@]}" rustfs/src/app/object_usecase.rs rustfs/src/app/object "${MULTIPART_USECASE_GUARD_SOURCES[@]}" "${LIFECYCLE_TRANSITION_TEST_GUARD_SOURCES[@]}" || true
rg -n --with-filename \
'use super::(?:\{[^}]*\b(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b|(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b)|super::(?:lifecycle|metadata_sys|object_lock|quota|replication|tagging|target|utils|versioning_sys|transition_api)::|super::super::(?:metadata_sys|lifecycle|target)::' \
rustfs/src/app/bucket_usecase.rs rustfs/src/app/object_usecase.rs rustfs/src/app/object rustfs/src/app/multipart_usecase.rs rustfs/src/app/lifecycle_transition_api_test.rs rustfs/src/app/capacity_dirty_scope_test.rs rustfs/src/app/context.rs rustfs/src/app/context/handles.rs rustfs/src/app/context/interfaces.rs rustfs/src/app/context/runtime_sources.rs || true
"${BUCKET_USECASE_GUARD_SOURCES[@]}" rustfs/src/app/object_usecase.rs rustfs/src/app/object "${MULTIPART_USECASE_GUARD_SOURCES[@]}" "${LIFECYCLE_TRANSITION_TEST_GUARD_SOURCES[@]}" rustfs/src/app/capacity_dirty_scope_test.rs rustfs/src/app/context.rs rustfs/src/app/context/handles.rs rustfs/src/app/context/interfaces.rs rustfs/src/app/context/runtime_sources.rs || true
}
) >"$RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE"
@@ -2277,6 +2304,7 @@ fi
rg -n --with-filename \
'(use crate::storage::(access|request_context|ecfs)|use crate::storage::\{[^}]*Storage(ObjectInfo|ObjectOptions|PutObjReader)|crate::storage::StorageObjectOptions)' \
rustfs/src/app/*_test.rs \
"${LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES[@]}" \
rustfs/src/admin/router.rs \
rustfs/src/admin/console.rs \
rustfs/src/admin/handlers/heal.rs \
@@ -4561,7 +4589,7 @@ fi
cd "$ROOT_DIR"
rg -n --with-filename 'crate::storage::core_storage_compat' \
rustfs/src/storage/mod.rs \
rustfs/src/storage/ecfs.rs \
"${ECFS_GUARD_SOURCES[@]}" \
rustfs/src/storage/ecfs_extend.rs \
rustfs/src/storage/access.rs \
rustfs/src/storage/head_prefix.rs \
@@ -4577,7 +4605,8 @@ fi
cd "$ROOT_DIR"
{
rg -n --with-filename 'crate::app::usecase_storage_compat' \
rustfs/src/app/*.rs || true
rustfs/src/app/*.rs "${BUCKET_USECASE_SPLIT_SOURCES[@]}" "${MULTIPART_USECASE_SPLIT_SOURCES[@]}" \
"${LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES[@]}" || true
rg -n --with-filename 'crate::admin::router_storage_compat' \
rustfs/src/admin/router.rs || true
rg -n --with-filename 'crate::storage::core_storage_compat' \
@@ -1,226 +0,0 @@
#!/usr/bin/env bash
# Keeps the `gateway-stack` cargo feature of the rustfs crate out of every release build while the
# gateway S3 stack is under construction (rustfs/backlog#2734 T2.5).
#
# Usage: scripts/check_gateway_stack_release_boundary.sh [--root DIR]
# scripts/check_gateway_stack_release_boundary.sh --self-test
#
# Three facts are checked, none of which needs Cargo:
# 1. rustfs/src/server/gateway_stack.rs refuses to compile without debug assertions, so a
# release-profile build that enables the feature fails in the compiler;
# 2. neither the `default` nor the `full` feature list of rustfs/Cargo.toml enables it, directly
# or through a feature that enables it, and the feature itself still exists (a guard whose
# subject is gone must fail, not skip);
# 3. no file that produces a release artifact names the feature or one that enables it.
# A missing input is an error: every file below exists in this repository.
set -euo pipefail
FEATURE="gateway-stack"
GUARD_SOURCE="rustfs/src/server/gateway_stack.rs"
GUARD_CFG='#[cfg(not(debug_assertions))]'
GUARD_ERROR='compile_error!("gateway-stack is restricted to debug builds");'
MANIFEST="rustfs/Cargo.toml"
# Everything that builds or packages a release binary.
RELEASE_FILES=(
".github/workflows/build.yml"
".github/workflows/docker.yml"
"build-rustfs.sh"
"docker-buildx.sh"
"Dockerfile"
"Dockerfile.glibc"
"Dockerfile.source"
"rustfs.spec"
"Justfile"
".config/make/build.mak"
".config/make/build-docker.mak"
)
fail() {
echo "gateway-stack release boundary: $*" >&2
return 1
}
# Prints the value of one `name = [...]` feature line of the manifest.
feature_list() {
local manifest="$1" name="$2"
grep -E "^${name}[[:space:]]*=" "$manifest" || true
}
# Prints the feature and every feature that enables it, directly or through another, one per line.
enabling_features() {
local manifest="$1" known="$FEATURE" grown name
while true; do
grown="$known"
while IFS= read -r name; do
grep -qxF -- "$name" <<<"$grown" || grown+=$'\n'"$name"
done < <(
while IFS= read -r name; do
grep -E "^[A-Za-z0-9_-]+[[:space:]]*=.*\"${name}\"" "$manifest" | sed -E 's/[[:space:]]*=.*//' || true
done <<<"$known"
)
[[ "$grown" == "$known" ]] && break
known="$grown"
done
printf '%s\n' "$known"
}
check_root() {
local root="$1" status=0 file line name enabler
local -a enablers=("$FEATURE")
file="${root}/${GUARD_SOURCE}"
if [[ ! -f "$file" ]]; then
fail "${GUARD_SOURCE} is missing" || status=1
else
# The cfg must sit on the line directly above the compile_error.
line="$(grep -nF "$GUARD_ERROR" "$file" | head -n 1 | cut -d: -f1)"
if [[ -z "$line" ]]; then
fail "${GUARD_SOURCE} no longer refuses a build without debug assertions" || status=1
elif [[ "$(sed -n "$((line - 1))p" "$file" | tr -d '[:space:]')" != "$(tr -d '[:space:]' <<<"$GUARD_CFG")" ]]; then
fail "the compile_error in ${GUARD_SOURCE} is not gated on cfg(not(debug_assertions))" || status=1
fi
fi
file="${root}/${MANIFEST}"
if [[ ! -f "$file" ]]; then
fail "${MANIFEST} is missing" || status=1
else
if [[ -z "$(feature_list "$file" "$FEATURE")" ]]; then
fail "${MANIFEST} no longer declares the ${FEATURE} feature this guard protects" || status=1
fi
mapfile -t enablers < <(enabling_features "$file")
for name in default full; do
line="$(feature_list "$file" "$name")"
if [[ -z "$line" ]]; then
fail "${MANIFEST} has no ${name} feature list" || status=1
fi
for enabler in "${enablers[@]}"; do
if [[ "$enabler" == "$name" ]]; then
fail "the ${name} feature list of ${MANIFEST} enables ${FEATURE}" || status=1
fi
done
done
fi
for file in "${RELEASE_FILES[@]}"; do
if [[ ! -f "${root}/${file}" ]]; then
fail "release file ${file} is missing; update RELEASE_FILES" || status=1
continue
fi
for enabler in "${enablers[@]}"; do
if grep -qF -- "$enabler" "${root}/${file}"; then
fail "${file} names ${enabler}, which enables ${FEATURE}; release artifacts must not" || status=1
fi
done
done
return "$status"
}
self_test() {
local scratch fixture failures=0
scratch="$(mktemp -d)"
make_fixture() {
fixture="${scratch}/$1"
mkdir -p "${fixture}/rustfs/src/server" "${fixture}/.github/workflows" "${fixture}/.config/make"
printf '%s\n%s\n' "$GUARD_CFG" "$GUARD_ERROR" >"${fixture}/${GUARD_SOURCE}"
printf '%s\n' '[features]' 'default = ["ftps"]' 'full = ["ftps", "swift"]' 'gateway-stack = ["dep:rustfs-gateway"]' \
>"${fixture}/${MANIFEST}"
for file in "${RELEASE_FILES[@]}"; do
printf 'cargo build --release -p rustfs\n' >"${fixture}/${file}"
done
}
expect() {
local wanted="$1" label="$2"
if check_root "$fixture" >/dev/null 2>&1; then
[[ "$wanted" == "pass" ]] || { echo "self-test: ${label} was accepted" >&2; failures=$((failures + 1)); }
else
[[ "$wanted" == "fail" ]] || { echo "self-test: ${label} was refused" >&2; failures=$((failures + 1)); }
fi
}
make_fixture valid
expect pass "a repository that keeps the boundary"
make_fixture no-compile-error
printf '// nothing here\n' >"${fixture}/${GUARD_SOURCE}"
expect fail "a gateway stack without the compile-time refusal"
make_fixture ungated-compile-error
printf '%s\n%s\n' '#[cfg(test)]' "$GUARD_ERROR" >"${fixture}/${GUARD_SOURCE}"
expect fail "a compile_error gated on something else"
make_fixture missing-source
rm "${fixture}/${GUARD_SOURCE}"
expect fail "a missing guard source"
make_fixture default-enables
sed -i.bak 's/^default = .*/default = ["ftps", "gateway-stack"]/' "${fixture}/${MANIFEST}"
expect fail "a default feature list that enables the stack"
make_fixture full-enables
sed -i.bak 's/^full = .*/full = ["ftps", "gateway-stack"]/' "${fixture}/${MANIFEST}"
expect fail "a full feature list that enables the stack"
make_fixture feature-gone
sed -i.bak '/^gateway-stack = /d' "${fixture}/${MANIFEST}"
expect fail "a manifest without the feature"
make_fixture full-enables-through-another
printf '%s\n' 'hooks = ["gateway-stack"]' >>"${fixture}/${MANIFEST}"
sed -i.bak 's/^full = .*/full = ["ftps", "hooks"]/' "${fixture}/${MANIFEST}"
expect fail "a full feature list that enables the stack through another feature"
make_fixture debug-only-enabler
printf '%s\n' 'hooks = ["gateway-stack"]' >>"${fixture}/${MANIFEST}"
expect pass "a feature outside default and full that enables the stack"
make_fixture release-file-names-enabler
printf '%s\n' 'hooks = ["gateway-stack"]' >>"${fixture}/${MANIFEST}"
printf 'cargo build --release -p rustfs --features hooks\n' >"${fixture}/build-rustfs.sh"
expect fail "a release file that enables the stack through another feature"
make_fixture release-workflow
printf 'cargo build --release -p rustfs --features gateway-stack\n' >"${fixture}/.github/workflows/build.yml"
expect fail "a release workflow that enables the stack"
make_fixture missing-release-file
rm "${fixture}/build-rustfs.sh"
expect fail "a missing release file"
rm -rf "$scratch"
if ((failures > 0)); then
echo "gateway-stack release boundary self-test: ${failures} failure(s)" >&2
return 1
fi
echo "gateway-stack release boundary self-test passed"
}
main() {
local root
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
case "${1:-}" in
--self-test)
[[ $# -eq 1 ]] || { echo "usage: $0 [--root DIR] | --self-test" >&2; exit 2; }
self_test
;;
--root)
[[ $# -eq 2 ]] || { echo "usage: $0 [--root DIR] | --self-test" >&2; exit 2; }
check_root "$2"
echo "gateway-stack release boundary holds"
;;
"")
check_root "$root"
echo "gateway-stack release boundary holds"
;;
*)
echo "usage: $0 [--root DIR] | --self-test" >&2
exit 2
;;
esac
}
main "$@"
+69 -4
View File
@@ -22,7 +22,8 @@ classify_source_layer() {
printf 'composition'
elif [[ "$file" == rustfs/src/app/* ]]; then
printf 'app'
elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]] ||
elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] ||
[[ "$file" == rustfs/src/storage/legacy_edge/* ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]] ||
[[ "$file" == rustfs/src/storage/s3_compat/* ]]; then
printf 'interface'
elif [[ "$file" == rustfs/src/* ]]; then
@@ -55,6 +56,7 @@ classify_target_layer() {
storage)
storage_path="${path#storage::}"
if [[ "$storage_path" == "ecfs" ]] || [[ "$storage_path" == ecfs::* ]] ||
[[ "$storage_path" == "legacy_edge" ]] || [[ "$storage_path" == legacy_edge::* ]] ||
[[ "$storage_path" == "s3_api" ]] || [[ "$storage_path" == s3_api::* ]] ||
[[ "$storage_path" == "s3_compat" ]] || [[ "$storage_path" == s3_compat::* ]]; then
printf 'interface'
@@ -137,6 +139,60 @@ run_layer_model_self_tests() {
assert_dependency_direction 'allowed' "$s3_compat_source" "$app_target"
assert_dependency_direction 'reverse' "$app_source" "$s3_compat_target"
assert_dependency_direction 'reverse' "$storage_source" "$s3_compat_target"
# A mechanical split moves ecfs.rs code into storage/legacy_edge/ and the
# usecase files' code into same-named directories; each keeps its layer.
assert_source_layer 'interface' rustfs/src/storage/ecfs.rs
assert_source_layer 'interface' rustfs/src/storage/legacy_edge/bucket_core.rs
assert_source_layer 'interface' rustfs/src/storage/legacy_edge/tests/object_meta.rs
assert_source_layer 'infra' rustfs/src/storage/legacy_edge.rs
assert_source_layer 'infra' rustfs/src/storage/legacy_edge_helper.rs
assert_source_layer 'app' rustfs/src/app/bucket_usecase/cors.rs
assert_source_layer 'app' rustfs/src/app/multipart_usecase/complete.rs
assert_target_layer 'interface' storage::legacy_edge
assert_target_layer 'interface' storage::legacy_edge::object_meta
assert_target_layer 'infra' storage::legacy_edge_helper
assert_dependency_direction 'allowed' "$(classify_source_layer rustfs/src/storage/legacy_edge/bucket_core.rs)" "$app_target"
assert_dependency_direction 'reverse' "$app_source" "$(classify_target_layer storage::legacy_edge::bucket_core)"
assert_dependency_direction 'reverse' "$storage_source" "$(classify_target_layer storage::legacy_edge)"
assert_dedicated_test_file 'yes' rustfs/src/app/lifecycle_transition_api_test.rs
assert_dedicated_test_file 'yes' rustfs/src/app/lifecycle_transition_api_test/put.rs
assert_dedicated_test_file 'no' rustfs/src/app/lifecycle_transition_api_test_support.rs
assert_dedicated_test_file 'no' rustfs/src/app/lifecycle_transition_api.rs
assert_dedicated_test_file 'no' rustfs/src/app/bucket_usecase/cors.rs
assert_dedicated_test_file 'no' rustfs/src/storage/legacy_edge/bucket_core.rs
}
assert_source_layer() {
local expected="$1" file="$2" actual
actual="$(classify_source_layer "$file")"
if [[ "$actual" != "$expected" ]]; then
printf 'Layer dependency guard self-test failed: %s classified as %s (expected %s)\n' "$file" "$actual" "$expected" >&2
exit 1
fi
}
assert_target_layer() {
local expected="$1" path="$2" actual
actual="$(classify_target_layer "$path")"
if [[ "$actual" != "$expected" ]]; then
printf 'Layer dependency guard self-test failed: crate::%s classified as %s (expected %s)\n' "$path" "$actual" "$expected" >&2
exit 1
fi
}
assert_dedicated_test_file() {
local expected="$1" file="$2" actual='no'
if is_dedicated_test_file "$file"; then
actual='yes'
fi
if [[ "$actual" != "$expected" ]]; then
printf 'Layer dependency guard self-test failed: %s dedicated test file = %s (expected %s)\n' "$file" "$actual" "$expected" >&2
exit 1
fi
}
normalize_import_group_item() {
@@ -251,11 +307,20 @@ normalize_import_path() {
printf '%s\n' "$path"
}
# The guard is file-scoped: dedicated test modules are excluded, while inline
# #[cfg(test)] imports remain subject to the source file's layer. The
# lifecycle_transition_api_test/ directory is where a mechanical split of
# lifecycle_transition_api_test.rs moves its tests (rustfs/backlog#2765).
is_dedicated_test_file() {
local file="$1"
[[ "$file" == *_test.rs ]] || [[ "$file" == */tests/* ]] ||
[[ "$file" == rustfs/src/app/lifecycle_transition_api_test/* ]]
}
emit_crate_use_statements() {
(cd "$ROOT_DIR" && rg --files -g '*.rs' rustfs/src | while IFS= read -r file; do
# The guard is file-scoped: dedicated test modules are excluded, while
# inline #[cfg(test)] imports remain subject to the source file's layer.
if [[ "$file" == *_test.rs ]] || [[ "$file" == */tests/* ]]; then
if is_dedicated_test_file "$file"; then
continue
fi
perl -0777 -ne '
+19 -1
View File
@@ -1094,11 +1094,29 @@ trace_hot_spans=(
"rustfs/src/app/object:execute_get_object"
)
# A hot span in a file that is due for a mechanical split (rustfs/backlog#2765)
# is also looked up in the directory the split moves its functions into; the
# named file itself stays required.
# shellcheck source=scripts/guard_source_set.sh
source "${repo_root}/scripts/guard_source_set.sh"
for hot_span in "${trace_hot_spans[@]}"; do
file="${hot_span%%:*}"
function="${hot_span##*:}"
hot_span_sources=("$file")
split_dir=""
case "$file" in
rustfs/src/app/bucket_usecase.rs) split_dir="rustfs/src/app/bucket_usecase" ;;
esac
if [[ -n "$split_dir" ]]; then
if ! guard_source_set "$repo_root" "$file" "$split_dir"; then
echo "❌ logging guardrail violation: $file::$function guard source is missing" >&2
exit 1
fi
hot_span_sources=("${GUARD_SOURCES[@]}")
fi
trace_span_pattern="#\\[(tracing::)?instrument\\([^]]*level = \\\"trace\\\"[^]]*\\)\\]([[:space:]]+#\\[[^]]+\\])*[[:space:]]+(pub(\\([^)]*\\))?[[:space:]]+)?(super[[:space:]]+)?async fn ${function}\\b"
if ! rg -U "$trace_span_pattern" "$file" >/dev/null; then
if ! rg -U "$trace_span_pattern" "${hot_span_sources[@]}" >/dev/null; then
echo "❌ logging guardrail violation: $file::$function must remain TRACE-only" >&2
exit 1
fi
+2 -4
View File
@@ -17,7 +17,7 @@ ROOT = Path(__file__).resolve().parent.parent
ALWAYS_JOBS = ("classify-changes", "typos", "quick-checks")
CODE_JOBS = (
"workspace-lint-and-build", "workspace-tests", "core-contract", "connect-feature-boundaries", "fips-provider", "test-and-lint-rio-v2",
"test-and-lint-protocols", "test-and-lint-gateway-stack", "build-rustfs-debug-binary", "uring-integration",
"test-and-lint-protocols", "build-rustfs-debug-binary", "uring-integration",
"e2e-tests", "s3-implemented-tests", "s3-lifecycle-behavior-tests",
)
# Specialized boundary, hardware and alternate-feature lanes do not gate a
@@ -33,7 +33,7 @@ NIGHTLY_JOBS = ("connect-feature-boundaries", "fips-provider", "uring-integratio
# against this revision's expected job contract. The workflow refuses a plan
# that does not carry the current marker and falls back to the full matrix, so
# a base planner predating the contract cannot authorize a narrower selection.
PLAN_CONTRACT = "4"
PLAN_CONTRACT = "3"
# These jobs still exercise the server and its black-box harness when only E2E
# Rust sources change. Production code, manifests and shared test configuration
# always select the full matrix. No production package depends on e2e_test.
@@ -299,8 +299,6 @@ def configurations(jobs: dict[str, str]) -> list[dict]:
("protocol-sftp-tests", "test-and-lint-protocols", "test", ["sftp"]),
("protocol-http3-lint", "test-and-lint-protocols", "dev", ["http3", "swift"]),
("protocol-http3-tests", "test-and-lint-protocols", "test", ["http3", "swift"]),
("gateway-stack-lint", "test-and-lint-gateway-stack", "dev", ["gateway-stack"]),
("gateway-stack-tests", "test-and-lint-gateway-stack", "test", ["gateway-stack"]),
("server-rio-v2", "build-rustfs-debug-binary-rio-v2", "dev", ["e2e-test-hooks", "rio-v2"]),
]
return [{"id": identity, "job": job, "target": "x86_64-unknown-linux-gnu", "profile": profile,
+994
View File
@@ -0,0 +1,994 @@
#!/usr/bin/env python3
"""Split rustfs/src/app/multipart_usecase.rs into one file per operation.
Preparatory refactor P3 of the s3s to rustfs-gateway migration. A pure code
move: every top-level item, every method of `impl DefaultMultipartUsecase` and
every test is assigned to a target file by the explicit tables below, and is
moved verbatim with its attributes and doc comments. Nothing is renamed and no
body changes, with these exceptions, each mechanical and listed here so a
reviewer knows where the output differs from the input:
* the three test fixtures (`s3_op_total`, `build_request`, `make_usecase`)
gain `pub(super)` because they move to `test_support.rs`;
* the test module's `use` items become `pub(super) use` re-exports in
`test_support.rs` (names the parent already imports are not repeated), so
every per-operation test module reaches them with one glob;
* code-position `s3s::` paths in the new files become
`crate::app::storage_api::s3::`, the sanctioned s3s funnel, so the number
of files naming s3s (scripts/check_s3s_footprint.sh) does not grow. The
funnel gains two re-exports, `dto` and `xml`, in rustfs/src/app/storage_api.rs;
* the nextest filter that names the one test it pins follows the test to its
new module path.
Layout produced (parent keeps the struct, its constructors and accessors, the
shared helpers and the test fixtures):
multipart_usecase.rs shared state and helpers
multipart_usecase/test_support.rs fixtures and test-only imports
multipart_usecase/{abort,create,list,upload_part,complete,upload_part_copy}.rs
one operation: its helpers, its
`impl DefaultMultipartUsecase` block,
its tests
multipart_usecase/upload_part/tests/body_read_tests.rs
multipart_usecase/upload_part_copy/tests/copy_admission_tests.rs
the two former test files, unchanged
Usage:
scripts/codemods/split_multipart_usecase.py [--no-fmt] [--report]
scripts/codemods/split_multipart_usecase.py --self-test
The input is the unsplit tree. A tree that is already split is refused, as is
any top-level item, method or test without an assignment, any table entry that
matches nothing, and any private item that a file other than its own refers
to. Re-running therefore means restoring the pre-split files first:
git checkout <pre-split-commit> -- rustfs/src/app/multipart_usecase.rs \\
rustfs/src/app/multipart_usecase rustfs/src/app/storage_api.rs .config/nextest.toml
git clean -fd rustfs/src/app/multipart_usecase
and then running this script. rustfmt formats the written files unless
--no-fmt is given. --report prints the item to file assignment and the
cross-file reference check and writes nothing.
Exit status: 0 on success, 1 when the input cannot be split as asked (the
message says why), 2 for a usage error.
"""
import argparse
import re
import subprocess
import sys
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
APP = "rustfs/src/app"
SOURCE = f"{APP}/multipart_usecase.rs"
OUT_DIR = f"{APP}/multipart_usecase"
STORAGE_API = f"{APP}/storage_api.rs"
NEXTEST = ".config/nextest.toml"
STRUCT = "DefaultMultipartUsecase"
FUNNEL = "crate::app::storage_api::s3::"
PARENT = "parent"
SUPPORT = "test_support"
OPS = ("abort", "create", "list", "upload_part", "complete", "upload_part_copy")
OP_DOC = {
"abort": "AbortMultipartUpload.",
"create": "CreateMultipartUpload.",
"list": "ListMultipartUploads and ListParts.",
"upload_part": "UploadPart.",
"complete": "CompleteMultipartUpload.",
"upload_part_copy": "UploadPartCopy.",
}
# Test files that already existed, moved unchanged: (old path, operation).
OLD_TEST_FILES = {
"body_read_tests": (f"{OUT_DIR}/tests/body_read_tests.rs", "upload_part"),
"copy_admission_tests": (f"{OUT_DIR}/tests/copy_admission_tests.rs", "upload_part_copy"),
}
# ---------------------------------------------------------------------------
# Assignment tables: item name -> target file. Every name must be listed; the
# script fails on a missing or an unused entry.
# ---------------------------------------------------------------------------
def assigned(target, names):
return {name: target for name in names.split()}
# Top-level functions (everything that is not an import, the struct, its impl
# or the test module). A helper stays in the parent when more than one
# operation, or a cross-operation test, uses it.
FUNCTIONS = {
**assigned(
PARENT,
"""
merge_part_encryption_metadata multipart_plaintext_size multipart_part_numbers
multipart_max_total_object_size validate_table_catalog_object_mutation
internal_object_info_lookup_opts multipart_upload_api_error
""",
),
**assigned(
"upload_part_copy",
"normalize_copy_source_version_id format_copy_source_version_id validate_copy_source_range_not_exceeds",
),
**assigned(
"complete",
"""
validate_complete_multipart_parts normalize_complete_multipart_parts complete_part_from_s3
has_complete_multipart_object_lock_headers validate_complete_multipart_checksum_type
quota_accounting_object_size encode_s3_path extract_request_scheme extract_request_host
build_complete_multipart_location
""",
),
**assigned("create", "create_multipart_upload_metadata should_advertise_session_compression"),
**assigned(
"upload_part",
"""
resolve_upload_part_size require_upload_part_size upload_part_body_read_timeout
passthrough_part_actual_size normalize_presigned_part_checksums
""",
),
}
# Methods of `impl DefaultMultipartUsecase`.
METHODS = {
**assigned(
PARENT,
"without_context from_global with_context with_context_and_concurrency_manager"
" bucket_metadata_sys object_store object_data_cache concurrency_manager",
),
**assigned("abort", "execute_abort_multipart_upload"),
**assigned("complete", "execute_complete_multipart_upload execute_complete_multipart_upload_owned"),
**assigned("create", "execute_create_multipart_upload"),
**assigned("upload_part", "execute_upload_part"),
**assigned("list", "execute_list_multipart_uploads execute_list_parts"),
**assigned("upload_part_copy", "execute_upload_part_copy"),
}
# Items of `mod tests`. A test that drives more than one operation, and the
# tests of the shared helpers, stay in the parent's own `mod tests`.
TESTS = {
**assigned(SUPPORT, "s3_op_total build_request make_usecase"),
**assigned(
PARENT,
"""
internal_object_info_lookup_opts_drops_http_preconditions
merge_part_encryption_metadata_keeps_source_metadata_unchanged
managed_multipart_roundtrip_preserves_session_nonce_between_parts
multipart_upload_rejects_reserved_directory_markers
multipart_max_total_object_size_reads_compatible_internal_metadata
multipart_upload_api_error_reports_malformed_upload_id_as_no_such_upload
multipart_handlers_reject_another_callers_denied_kms_upload
""",
),
**assigned("abort", "execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized"),
**assigned(
"create",
"""
session_compression_is_advertised_only_for_non_passthrough_compressible_uploads
execute_create_multipart_upload_rejects_invalid_storage_class
create_multipart_upload_metadata_persists_dto_storage_class_without_raw_header
create_multipart_upload_metadata_keeps_user_and_system_namespaces_separate
create_multipart_rejects_ciphertext_replication_before_parts_are_staged
""",
),
**assigned(
"list",
"""
execute_list_multipart_uploads_returns_internal_error_when_store_uninitialized
execute_list_multipart_uploads_rejects_invalid_key_marker_before_store_lookup
execute_list_multipart_uploads_rejects_invalid_max_uploads_before_store_lookup
execute_list_parts_returns_internal_error_when_store_uninitialized
execute_list_parts_rejects_negative_part_number_marker_before_store_lookup
execute_list_parts_rejects_invalid_max_parts_before_store_lookup
""",
),
**assigned(
"upload_part",
"""
presigned_checksum_request
normalize_presigned_part_checksums_validates_body
normalize_presigned_part_checksums_accepts_sdk_crc32_default
normalize_presigned_part_checksums_rejects_ambiguous_values
normalize_presigned_part_checksums_preserves_headers_and_rejects_invalid_values
normalize_presigned_part_checksums_requires_verified_presign
resolve_upload_part_size_uses_normalized_logical_length
upload_part_length_contract_rejects_unknown_and_all_negative_lengths
upload_part_timeout_policy_preserves_disabled_and_capped_floor
execute_upload_part_rejects_missing_body
execute_upload_part_rejects_oversize_declared_part_before_reading_the_body
execute_upload_part_rejects_invalid_part_number_before_body_lookup
execute_upload_part_rejects_unknown_length_before_admission_or_body_polling
execute_upload_part_progress_timeout_releases_admission_and_leaves_no_part
execute_upload_part_rejects_when_foreground_write_admission_is_full
""",
),
**assigned(
"upload_part_copy",
"""
execute_upload_part_copy_returns_internal_error_when_store_uninitialized
execute_upload_part_copy_rejects_invalid_part_number_before_store_lookup
test_validate_copy_source_range_not_exceeds_returns_invalid_range_when_range_exceeds
test_validate_copy_source_range_not_exceeds_ok_when_range_valid
test_validate_copy_source_range_not_exceeds_ok_for_suffix_range
upload_part_copy_normalizes_and_formats_source_version_id
""",
),
**assigned(
"complete",
"""
upload_metadata_with_checksum_type headers_with_checksum_type
complete_multipart_checksum_type_matching_the_upload_is_accepted
complete_multipart_checksum_type_contradicting_the_upload_is_rejected
complete_multipart_checksum_type_without_both_sides_is_accepted
quota_accounting_uses_logical_size_when_available
quota_accounting_fails_closed_only_when_quota_is_configured
test_build_complete_multipart_location_uses_forwarded_proto_and_encodes_key
test_build_complete_multipart_location_falls_back_to_uri_authority_and_scheme
test_build_complete_multipart_location_returns_path_without_host
execute_complete_multipart_upload_rejects_missing_parts_payload
execute_complete_multipart_upload_rejects_missing_parts_list
compressed_complete_records_logical_quota_usage_and_overwrite_delta
authorized_replica_only_multipart_complete_preserves_persisted_anti_cascade_state
authorized_replica_only_multipart_complete_preserves_persisted_anti_cascade_state_inner
ssec_completion_and_replay_require_key_before_disclosing_plaintext_checksums
assert_multipart_etag_listings
kms_multipart_completion_and_replay_preserve_etag_in_versioned_listings
conditional_multipart_upload_if_none_match_returns_precondition_failed_when_object_exists
conditional_multipart_upload_if_none_match_returns_precondition_failed_when_object_exists_inner
concurrent_completions_share_durable_bucket_quota_reservations
multipart_completion_commits_after_quota_fleet_proof_loss_before_rename
data_movement_multipart_completion_has_zero_quota_growth
cancelled_multipart_completion_emits_once_and_replay_does_not_emit
rejected_empty_parts_preserve_existing_object_and_staging
execute_complete_multipart_upload_allows_duplicate_part_numbers_by_using_last_occurrence
execute_complete_multipart_upload_rejects_out_of_order_parts
validate_complete_multipart_parts_enforces_part_number_range
normalize_complete_multipart_parts_keeps_last_duplicate_part
execute_complete_multipart_upload_rejects_object_lock_headers
""",
),
}
# Test module items that are not code to assign.
TEST_MODS = ("body_read_tests", "copy_admission_tests")
CODE, COMMENT, LITERAL = 0, 1, 2
IDENT_CHARS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_")
CHAR_LITERAL = re.compile(r"'(?:\\(?:[nrt\\0'\"]|x[0-9a-fA-F]{2}|u\{[0-9a-fA-F_]{1,6}\})|[^\\'\n])'")
RAW_STRING_START = re.compile(r'r(#*)"')
# A relative path in code moved one module down: `super::object::..` named a sibling of
# multipart_usecase, and from an operation file that is `super::super::object::..`.
SUPER_PATH = re.compile(r"(?<![A-Za-z0-9_:])super::(?!\*)")
S3S_PATH = re.compile(r"(?<![\"A-Za-z0-9_])s3s::")
VIS = r"(?:pub(?:\([^)]*\))?\s+)?"
FN_HEAD = re.compile(r"^" + VIS + r"(?:const\s+)?(?:async\s+)?(?:unsafe\s+)?fn\s+([A-Za-z_][A-Za-z0-9_]*)")
STRUCT_HEAD = re.compile(r"^" + VIS + r"struct\s+([A-Za-z_][A-Za-z0-9_]*)")
IMPL_HEAD = re.compile(r"^impl\s+([A-Za-z_][A-Za-z0-9_]*)\s*\{")
MOD_BLOCK_HEAD = re.compile(r"^" + VIS + r"mod\s+([A-Za-z_][A-Za-z0-9_]*)\s*\{")
MOD_DECL_HEAD = re.compile(r"^" + VIS + r"mod\s+([A-Za-z_][A-Za-z0-9_]*)\s*;")
USE_HEAD = re.compile(r"^" + VIS + r"use\s")
class SplitError(Exception):
"""The input cannot be split as asked: exit 1."""
# ---------------------------------------------------------------------------
# Lexing: blank comments and literals so structure can be read from the rest.
# ---------------------------------------------------------------------------
def classify(text):
"""Return one kind byte per character: CODE, COMMENT or LITERAL."""
kinds = bytearray(len(text))
i, n = 0, len(text)
def mark(start, end, kind):
kinds[start:end] = bytes([kind]) * (end - start)
while i < n:
ch = text[i]
if text.startswith("//", i):
end = text.find("\n", i)
end = n if end < 0 else end
mark(i, end, COMMENT)
i = end
elif text.startswith("/*", i):
depth, j = 1, i + 2
while j < n and depth:
if text.startswith("/*", j):
depth, j = depth + 1, j + 2
elif text.startswith("*/", j):
depth, j = depth - 1, j + 2
else:
j += 1
mark(i, j, COMMENT)
i = j
elif ch == "r" and (i == 0 or text[i - 1] not in IDENT_CHARS or (text[i - 1] == "b" and (i == 1 or text[i - 2] not in IDENT_CHARS))):
raw = RAW_STRING_START.match(text, i)
if raw is None:
i += 1
continue
closing = '"' + raw.group(1)
end = text.find(closing, raw.end())
end = n if end < 0 else end + len(closing)
mark(i, end, LITERAL)
i = end
elif ch == '"':
j = i + 1
while j < n and text[j] != '"':
j += 2 if text[j] == "\\" else 1
mark(i, min(j + 1, n), LITERAL)
i = j + 1
elif ch == "'":
literal = CHAR_LITERAL.match(text, i)
if literal is None: # a lifetime or a loop label
i += 1
continue
mark(i, literal.end(), LITERAL)
i = literal.end()
else:
i += 1
return kinds
def code_only(text):
"""`text` with comments and literals replaced by spaces, newlines kept."""
kinds = classify(text)
return "".join(ch if kinds[i] == CODE or ch == "\n" else " " for i, ch in enumerate(text))
# ---------------------------------------------------------------------------
# Item splitting. The input is rustfmt output, so an item is the run of lines
# from its first attribute or comment to the line on which brackets balance
# and the line ends in `}` or `;`.
# ---------------------------------------------------------------------------
class Item:
def __init__(self, lines, code_lines):
self.lines = lines # original lines, leading comments and attributes included
self.code_lines = code_lines # the same lines with comments and literals blanked
self.head = next((line.strip() for line in code_lines if line.strip() and not line.strip().startswith("#[")), "")
self.cfg_test = any(line.strip() == "#[cfg(test)]" for line in lines)
@property
def text(self):
return "\n".join(self.lines)
@property
def code(self):
return "\n".join(self.code_lines)
def name(self):
for pattern in (FN_HEAD, STRUCT_HEAD, IMPL_HEAD, MOD_BLOCK_HEAD, MOD_DECL_HEAD):
match = pattern.match(self.head)
if match:
return match.group(1)
return None
def is_use(self):
return bool(USE_HEAD.match(self.head))
def split_items(lines, code_lines, what):
"""Cut `lines` into items; blank lines separate them."""
items, i, n = [], 0, len(lines)
while i < n:
if not lines[i].strip():
i += 1
continue
start, balance, j = i, 0, i
while j < n:
stripped = code_lines[j].rstrip()
balance += sum(stripped.count(c) for c in "([{") - sum(stripped.count(c) for c in ")]}")
if balance == 0 and stripped.endswith(("}", ";")):
break
j += 1
if j >= n:
raise SplitError(f"{what}: an item starting at line {start + 1} never closes")
items.append(Item(lines[start : j + 1], code_lines[start : j + 1]))
i = j + 1
return items
def canonical(items, separator):
return separator.join(item.text for item in items)
# ---------------------------------------------------------------------------
# Use trees.
# ---------------------------------------------------------------------------
def split_top(inner):
parts, depth, current = [], 0, []
for ch in inner:
if ch == "{":
depth += 1
elif ch == "}":
depth -= 1
if ch == "," and depth == 0:
parts.append("".join(current))
current = []
else:
current.append(ch)
parts.append("".join(current))
return [p.strip() for p in parts if p.strip()]
def flatten_use(tree, prefix=""):
"""[(full path, bound name)] for a use tree; a glob binds `*`, `as _` binds `_`."""
tree = tree.strip()
brace = tree.find("{")
if brace >= 0:
head = tree[:brace].strip()
inner = tree[brace + 1 : tree.rindex("}")]
base = prefix + head
found = []
for part in split_top(inner):
found.extend(flatten_use(part, base))
return found
alias = None
if " as " in tree:
tree, alias = (side.strip() for side in tree.split(" as ", 1))
path = prefix + tree
last = path.split("::")[-1]
if last == "self":
path = path[: -len("::self")]
last = path.split("::")[-1]
return [(path, alias if alias else last)]
def use_paths(item):
body = re.sub(r"^\s*(?:#\[[^\n]*\]\s*)*", "", item.code, flags=re.S)
body = re.sub(r"^" + VIS + r"use\s+", "", body.strip())
body = body.rstrip().rstrip(";")
return flatten_use(body)
# ---------------------------------------------------------------------------
# The parsed input.
# ---------------------------------------------------------------------------
class Parsed:
pass
def parse(text):
lines = text.split("\n")
if lines and lines[-1] == "":
lines.pop()
code_lines = code_only("\n".join(lines)).split("\n")
# Leading licence and `//!` lines are the file preamble.
first = 0
while first < len(lines) and (lines[first].startswith("//") or not lines[first].strip()):
first += 1
preamble = lines[:first]
items = split_items(lines[first:], code_lines[first:], "multipart_usecase.rs")
parsed = Parsed()
parsed.preamble = preamble
parsed.imports, parsed.functions, parsed.body = [], [], []
parsed.struct = parsed.impl = parsed.tests = None
for item in items:
if not item.is_use():
parsed.body.append(item)
name = item.name()
if item.is_use():
if parsed.functions or parsed.struct or parsed.impl:
raise SplitError("a `use` item follows code: imports are expected first")
parsed.imports.append(item)
elif STRUCT_HEAD.match(item.head) and name == STRUCT:
parsed.struct = item
elif IMPL_HEAD.match(item.head) and name == STRUCT:
if parsed.impl is not None:
raise SplitError(f"more than one `impl {STRUCT}` block")
parsed.impl = item
elif MOD_BLOCK_HEAD.match(item.head) and name == "tests":
parsed.tests = item
elif FN_HEAD.match(item.head):
item.after_impl = parsed.impl is not None
parsed.functions.append(item)
elif MOD_DECL_HEAD.match(item.head):
raise SplitError(f"{SOURCE} already declares {item.head!r}: it is split already; restore the pre-split files first")
else:
raise SplitError(f"unrecognised top-level item: {item.head!r}")
for what, value in (("struct", parsed.struct), ("impl", parsed.impl), ("tests module", parsed.tests)):
if value is None:
raise SplitError(f"the {what} was not found; is the input already split?")
# Spacing must be the canonical one, or the output would not be a move.
rebuilt = "\n".join(preamble) + "\n" + "\n".join(i.text for i in parsed.imports) + "\n\n"
rebuilt += canonical(parsed.body, "\n\n")
if "\n".join(lines) != "\n".join(rebuilt.split("\n")):
raise SplitError("the item layout is not the one this script reassembles (spacing differs from rustfmt's)")
parsed.impl_methods = body_items(parsed.impl, "impl")
parsed.test_items = body_items(parsed.tests, "mod tests")
return parsed
def body_items(block, what):
"""Items between the opening line and the closing brace of `block`."""
opening = next(i for i, line in enumerate(block.code_lines) if line.strip() and not line.strip().startswith("#["))
inner, inner_code = block.lines[opening + 1 : -1], block.code_lines[opening + 1 : -1]
items = split_items(inner, inner_code, what)
return items
# ---------------------------------------------------------------------------
# Assignment and checks.
# ---------------------------------------------------------------------------
def assign(items, table, what):
placed = {}
seen = set()
for item in items:
name = item.name()
if name is None:
raise SplitError(f"{what}: cannot name the item {item.head!r}")
if name in seen:
raise SplitError(f"{what}: two items are named {name}")
seen.add(name)
target = table.get(name)
if target is None:
raise SplitError(f"{what}: no assignment for {name} ({item.head[:60]!r})")
placed[name] = (target, item)
unused = sorted(set(table) - seen)
if unused:
raise SplitError(f"{what}: assignments that match nothing: {', '.join(unused)}")
return placed
def check_targets(table, what, allowed):
for name, target in table.items():
if target not in allowed:
raise SplitError(f"{what}: {name} is assigned to unknown file {target!r}")
def word(name):
return re.compile(r"(?<![A-Za-z0-9_])" + re.escape(name) + r"(?![A-Za-z0-9_])")
def referenced_from(name, texts, own):
"""Files other than `own` whose code mentions `name`."""
pattern = word(name)
return sorted(f for f, code in texts.items() if f != own and pattern.search(code))
def plan(parsed):
"""Decide the content of every output file; returns (pieces, report)."""
check_targets(FUNCTIONS, "functions", {PARENT, *OPS})
check_targets(METHODS, "methods", {PARENT, *OPS})
check_targets(TESTS, "tests", {PARENT, SUPPORT, *OPS})
functions = assign(parsed.functions, FUNCTIONS, "functions")
methods = assign(parsed.impl_methods, METHODS, "methods")
test_code_items = []
test_uses, test_mods, test_glob = [], [], None
for item in parsed.test_items:
if item.is_use():
if item.head.replace(" ", "") == "usesuper::*;":
test_glob = item
else:
test_uses.append(item)
elif MOD_DECL_HEAD.match(item.head):
if item.name() not in TEST_MODS:
raise SplitError(f"mod tests: unexpected module declaration {item.head!r}")
test_mods.append(item)
else:
test_code_items.append(item)
if test_glob is None:
raise SplitError("mod tests does not start with `use super::*;`")
if sorted(m.name() for m in test_mods) != sorted(TEST_MODS):
raise SplitError("mod tests: the declared test modules differ from TEST_MODS")
tests = assign(test_code_items, TESTS, "tests")
for placed in (functions, methods):
for target, item in placed.values():
if target != PARENT:
push_down(item)
# Piece lists per target file.
pieces = {f: {"functions": [], "methods": [], "tests": []} for f in (PARENT, SUPPORT, *OPS)}
for _, (target, item) in functions.items():
pieces[target]["functions"].append(item)
for _, (target, item) in methods.items():
pieces[target]["methods"].append(item)
for _, (target, item) in tests.items():
pieces[target]["tests"].append(item)
if pieces[SUPPORT]["functions"] or pieces[SUPPORT]["methods"]:
raise SplitError("test_support holds tests-module fixtures only")
pieces["_uses"] = test_uses
pieces["_mods"] = test_mods
return pieces, (functions, methods, tests)
def per_file_code(parsed, pieces, extra):
"""Code of each output file for the reference check (parent keeps imports)."""
texts = {}
for target in (PARENT, SUPPORT, *OPS):
parts = []
if target == PARENT:
parts += [i.code for i in parsed.imports] + [parsed.struct.code]
parts += [i.code for i in pieces[target]["functions"]]
parts += [i.code for i in pieces[target]["methods"]]
parts += [i.code for i in pieces[target]["tests"]]
texts[target] = "\n".join(parts)
for target, code in extra.items():
texts[target] += "\n" + code
return texts
def reference_check(pieces, texts):
"""A private item moved out of the parent must stay inside its own file.
Items kept in the parent and fixtures in test_support are visible to every
child through the globs, so only items placed in an operation file are
checked. A name that is also a local binding elsewhere would show up here as
a false alarm; the message names the files so the table can be adjusted.
"""
problems = []
for group in ("functions", "methods", "tests"):
for target in OPS:
for item in pieces[target][group]:
name = item.name()
if name is None or not FN_HEAD.match(item.head):
continue
if re.match(r"^pub(\(|\s)", item.head) and group == "methods":
continue # a public method is reached through the type
users = referenced_from(name, texts, target)
if users:
problems.append(f"{name} is placed in {target} but also mentioned in {', '.join(users)}")
return problems
# ---------------------------------------------------------------------------
# Rendering.
# ---------------------------------------------------------------------------
def rewrite_code(text, pattern, replacement):
"""Replace `pattern` matches that sit in code, never in comments or literals."""
kinds = classify(text)
out, last = [], 0
for match in pattern.finditer(text):
if kinds[match.start()] != CODE:
continue
out.append(text[last : match.start()])
out.append(replacement)
last = match.end()
out.append(text[last:])
return "".join(out)
def rewrite_s3s(text):
"""Send code-position `s3s::` paths through the sanctioned funnel."""
return rewrite_code(text, S3S_PATH, FUNNEL)
def push_down(item):
"""Make the relative paths of an item that leaves the parent module point where they did."""
text = rewrite_code(item.text, SUPER_PATH, "super::super::")
item.lines = text.split("\n")
item.code_lines = code_only(text).split("\n")
def licence(preamble):
block = []
for line in preamble:
if not line.startswith("//") or line.startswith("//!"):
break
block.append(line)
return "\n".join(block)
def doc_line(preamble):
return next(line for line in preamble if line.startswith("//!"))
def join_items(items):
return "\n\n".join(item.text for item in items)
def render_operation(op, pieces, header, tests_extra_mods):
out = [header, f"//! {OP_DOC[op]}", "", "use super::*;", ""]
body = []
if pieces[op]["functions"]:
body.append(join_items(pieces[op]["functions"]))
if pieces[op]["methods"]:
body.append(f"impl {STRUCT} {{\n{join_items(pieces[op]['methods'])}\n}}")
if pieces[op]["tests"] or tests_extra_mods:
inner = [join_items(pieces[op]["tests"])] if pieces[op]["tests"] else []
inner += [f" mod {name};" for name in tests_extra_mods]
body.append(
"#[cfg(test)]\nmod tests {\n use super::super::test_support::*;\n use super::*;\n\n"
+ "\n\n".join(inner)
+ "\n}"
)
out.append("\n\n".join(body))
return rewrite_s3s("\n".join(out) + "\n")
def render_test_support(parsed, pieces, header, parent_bound):
out = [header, "//! Fixtures and test-only imports shared by the multipart use-case tests.", "", "use super::*;"]
reexports, notes = [], []
for item in pieces["_uses"]:
for path, name in use_paths(item):
if name in parent_bound:
notes.append((name, parent_bound[name], path))
continue
reexports.append(f"pub(super) use {path};")
out.extend(reexports)
out.append("")
fixtures = []
for item in pieces[SUPPORT]["tests"]:
lines = list(item.lines)
for index, line in enumerate(lines):
if FN_HEAD.match(line.strip()) and not line.startswith(" #"):
lines[index] = re.sub(r"^(\s*)", r"\1pub(super) ", line, count=1)
break
fixtures.append("\n".join(line[4:] if line.startswith(" ") else line for line in lines))
out.append("\n\n".join(fixtures))
return rewrite_s3s("\n".join(out) + "\n"), notes
def render_parent(parsed, pieces, header):
out = [header, "\n".join(line for line in parsed.preamble if line.startswith("//!")), ""]
out.append("\n".join(i.text for i in parsed.imports))
out.append("")
mods = [f"mod {op};" for op in sorted(OPS + (SUPPORT,))]
mods = [("#[cfg(test)]\n" + m) if SUPPORT in m else m for m in mods]
out.append("\n".join(mods))
out.append("")
before = [i for i in pieces[PARENT]["functions"] if not i.after_impl]
after = [i for i in pieces[PARENT]["functions"] if i.after_impl]
chunks = [join_items(before)] if before else []
chunks.append(
f"{parsed.struct.text}\n\n{parsed.impl.lines[0]}\n{join_items(pieces[PARENT]['methods'])}\n}}"
)
if after:
chunks.append(join_items(after))
if pieces[PARENT]["tests"]:
chunks.append(
rewrite_s3s(
"#[cfg(test)]\nmod tests {\n use super::test_support::*;\n use super::*;\n\n"
+ join_items(pieces[PARENT]["tests"])
+ "\n}"
)
)
out.append("\n\n".join(chunks))
return "\n".join(out) + "\n"
# ---------------------------------------------------------------------------
# Driver.
# ---------------------------------------------------------------------------
def bound_names(imports):
bound = {}
for item in imports:
for path, name in use_paths(item):
if name not in ("*", "_"):
bound[name] = path
return bound
def build(root):
source_path = root / SOURCE
if not source_path.is_file():
raise SplitError(f"{SOURCE} is missing")
text = source_path.read_text(encoding="utf-8")
parsed = parse(text)
pieces, (functions, methods, tests) = plan(parsed)
extra = {}
old_texts = {}
for name, (path, op) in OLD_TEST_FILES.items():
file = root / path
if not file.is_file():
raise SplitError(f"{path} is missing")
old_texts[name] = file.read_text(encoding="utf-8")
extra[op] = extra.get(op, "") + "\n" + code_only(old_texts[name])
texts = per_file_code(parsed, pieces, extra)
problems = reference_check(pieces, texts)
if problems:
raise SplitError("cross-file references:\n " + "\n ".join(problems))
header = licence(parsed.preamble) + "\n"
files = {}
parent_bound = bound_names(parsed.imports)
support, notes = render_test_support(parsed, pieces, header, parent_bound)
files[f"{OUT_DIR}/test_support.rs"] = support
for op in OPS:
mods = [name for name, (_, owner) in OLD_TEST_FILES.items() if owner == op]
files[f"{OUT_DIR}/{op}.rs"] = render_operation(op, pieces, header, mods)
for name, (path, op) in OLD_TEST_FILES.items():
files[f"{OUT_DIR}/{op}/tests/{name}.rs"] = old_texts[name]
files[SOURCE] = render_parent(parsed, pieces, header)
return parsed, pieces, files, notes, (functions, methods, tests)
def check_footprint(files):
for path, text in files.items():
if path == SOURCE:
continue
for number, line in enumerate(text.split("\n"), 1):
if S3S_PATH.search(line):
raise SplitError(f"{path}:{number} still names s3s directly: {line.strip()}")
return None
FUNNEL_NAMES = {"dto", "xml", "S3", "Body", "S3Request", "S3ErrorCode", "S3Response", "S3Error", "S3Result"}
def check_funnel_names(parsed_sources, files):
seen = set()
original = "\n".join(parsed_sources)
for match in S3S_PATH.finditer(original):
tail = re.match(r"[A-Za-z_][A-Za-z0-9_]*", original[match.end() :])
if tail:
seen.add(tail.group(0))
unknown = sorted(seen - FUNNEL_NAMES)
if unknown:
raise SplitError(f"s3s paths with no funnel re-export: {', '.join(unknown)}")
def edit_storage_api(text):
if "pub(crate) use s3s::dto;" in text:
raise SplitError(f"{STORAGE_API} already re-exports s3s::dto")
anchor = " #[cfg(test)]\n pub(crate) use s3s::dto::{GetObjectInput, HeadObjectInput, ListObjectsV2Input, ListObjectsV2Output};\n"
if text.count(anchor) != 1:
raise SplitError(f"{STORAGE_API}: the s3 module's dto re-export anchor was not found exactly once")
text = text.replace(anchor, " pub(crate) use s3s::dto;\n" + anchor)
anchor = " #[cfg(test)]\n pub(crate) use s3s::xml::{Serialize as XmlSerialize, Serializer as XmlSerializer};\n"
if text.count(anchor) != 1:
raise SplitError(f"{STORAGE_API}: the s3 module's xml re-export anchor was not found exactly once")
return text.replace(anchor, " #[cfg(test)]\n pub(crate) use s3s::xml;\n" + anchor)
def edit_nextest(text, tests):
"""Point the filter that pins one test at the module the test now lives in."""
replaced = 0
for name, (target, _) in tests.items():
needle = f"app::multipart_usecase::tests::{name}"
if needle in text:
moved = "app::multipart_usecase::tests" if target == PARENT else f"app::multipart_usecase::{target}::tests"
replaced += text.count(needle)
text = text.replace(needle, f"{moved}::{name}")
if replaced == 0:
raise SplitError(f"{NEXTEST} names no multipart_usecase test: has the filter moved?")
return text
def run(root, fmt, report):
parsed, pieces, files, notes, (functions, methods, tests) = build(root)
check_funnel_names([i.text for i in parsed.functions + parsed.impl_methods + parsed.test_items], files)
check_footprint(files)
if report:
print_report(parsed, pieces, files, notes, (functions, methods, tests))
return 0
storage = edit_storage_api((root / STORAGE_API).read_text(encoding="utf-8"))
nextest = edit_nextest((root / NEXTEST).read_text(encoding="utf-8"), tests)
for path, text in files.items():
target = root / path
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(text, encoding="utf-8")
for name, (path, op) in OLD_TEST_FILES.items():
(root / path).unlink()
old_dir = root / OUT_DIR / "tests"
if old_dir.is_dir() and not any(old_dir.iterdir()):
old_dir.rmdir()
(root / STORAGE_API).write_text(storage, encoding="utf-8")
(root / NEXTEST).write_text(nextest, encoding="utf-8")
if fmt:
written = [str(root / p) for p in files if p != SOURCE] + [str(root / SOURCE), str(root / STORAGE_API)]
result = subprocess.run(["rustfmt", "--edition", "2024", *written], check=False, cwd=root)
if result.returncode != 0:
raise SplitError("rustfmt failed on the written files")
summary(files, pieces)
return 0
def print_report(parsed, pieces, files, notes, assigned):
for target in (PARENT, SUPPORT, *OPS):
counts = {k: len(pieces[target][k]) for k in ("functions", "methods", "tests")}
print(f"{target}: {counts}")
for name, parent_path, test_path in notes:
print(f"note: test import {name} is already bound in the parent ({parent_path}); test path {test_path}")
for path, text in sorted(files.items()):
print(f"{path}: {len(text.splitlines())} lines")
def summary(files, pieces):
for path, text in sorted(files.items()):
print(f"wrote {path}: {len(text.splitlines())} lines")
for target in (PARENT, SUPPORT, *OPS):
fns = len(pieces[target]["functions"]) + len(pieces[target]["methods"])
tests = sum(1 for i in pieces[target]["tests"] if re.search(r"#\[(tokio::)?test", i.text))
print(f"{target}: {fns} production fns, {tests} tests")
def main(argv):
parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0])
parser.add_argument("--no-fmt", action="store_true", help="do not run rustfmt on the written files")
parser.add_argument("--report", action="store_true", help="print the assignment and write nothing")
parser.add_argument("--self-test", action="store_true", help="run the script's own tests")
args = parser.parse_args(argv)
if args.self_test:
suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTest)
return 0 if unittest.TextTestRunner(verbosity=1).run(suite).wasSuccessful() else 1
try:
return run(ROOT, not args.no_fmt, args.report)
except SplitError as error:
print(f"split_multipart_usecase: {error}", file=sys.stderr)
return 1
class SelfTest(unittest.TestCase):
SAMPLE = "\n".join(
[
"// licence",
"",
"//! doc",
"",
"use a::b;",
"",
"/// doc",
"fn one(",
" x: u8,",
") -> u8 {",
' let s = "}";',
" x",
"}",
"",
"#[cfg(test)]",
"mod tests {",
" use super::*;",
"",
" #[test]",
" fn t() {}",
"}",
]
)
def test_items_split_on_balance_not_on_braces_in_strings(self):
lines = self.SAMPLE.split("\n")[6:]
items = split_items(lines, code_only("\n".join(lines)).split("\n"), "sample")
self.assertEqual([i.name() for i in items], ["one", "tests"])
self.assertTrue(items[1].cfg_test)
def test_use_tree_flattening(self):
paths = flatten_use("a::{b, c::{d as e, f}, g::*}")
self.assertEqual(paths, [("a::b", "b"), ("a::c::d", "e"), ("a::c::f", "f"), ("a::g::*", "*")])
def test_s3s_rewrite_skips_strings_and_identifiers(self):
text = 'let a = s3s::dto::X; let b = "s3s::dto"; let c = rs3s::y;'
self.assertEqual(rewrite_s3s(text), f'let a = {FUNNEL}dto::X; let b = "s3s::dto"; let c = rs3s::y;')
def test_unassigned_item_fails(self):
lines = self.SAMPLE.split("\n")[6:13]
items = split_items(lines, code_only("\n".join(lines)).split("\n"), "sample")
with self.assertRaises(SplitError):
assign(items, {}, "functions")
with self.assertRaises(SplitError):
assign(items, {"one": PARENT, "ghost": PARENT}, "functions")
def test_unclosed_item_fails(self):
with self.assertRaises(SplitError):
split_items(["fn a() {"], ["fn a() {"], "sample")
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+51
View File
@@ -0,0 +1,51 @@
#!/usr/bin/env bash
#
# Source-set helper for text guards that must survive a mechanical file split.
# Sourced by check_architecture_migration_rules.sh and check_logging_guardrails.sh;
# contract tests: scripts/test_guard_source_set.sh.
#
# A guard that greps one named file stops seeing the code it guards once that
# file is split into a sibling directory. guard_source_set resolves the set of
# files such a guard is about instead:
#
# guard_source_set <root> <required-file> [<split-dir>...]
#
# On success it sets the global array GUARD_SOURCES to <required-file> followed
# by every `.rs` file under each <split-dir> that exists, in byte order of path
# (LC_ALL=C), all relative to <root>. An absent <split-dir> contributes nothing.
# It returns 1 with a message on stderr when <required-file> is not a regular
# file (a guard whose input is missing must fail, not pass) or when a
# <split-dir> exists but is not a directory. On failure GUARD_SOURCES still
# holds <required-file>, so a caller that records the failure and continues
# hands the missing path to its search tool rather than nothing.
guard_source_set() {
local root="$1" required="$2" split_dir path listing
shift 2
GUARD_SOURCES=("$required")
if [[ ! -f "${root}/${required}" ]]; then
printf 'guard source %s is missing\n' "$required" >&2
return 1
fi
for split_dir in "$@"; do
if [[ ! -e "${root}/${split_dir}" ]]; then
continue
fi
if [[ ! -d "${root}/${split_dir}" ]]; then
printf 'guard split path %s is not a directory\n' "$split_dir" >&2
return 1
fi
if ! listing="$(set -o pipefail; cd "$root" && find "$split_dir" -type f -name '*.rs' | LC_ALL=C sort)"; then
printf 'guard split directory %s cannot be listed\n' "$split_dir" >&2
return 1
fi
while IFS= read -r path; do
if [[ -n "$path" ]]; then
GUARD_SOURCES+=("$path")
fi
done <<<"$listing"
done
return 0
}
+3 -1
View File
@@ -509,7 +509,9 @@ pass/fail table in the job summary.
incomplete parameterized cases.
- `api_coverage.py` — quantifies S3 API surface coverage by comparing the
s3s `S3` trait (at the revision pinned in Cargo.toml) against the methods
RustFS overrides in `impl S3 for FS`:
RustFS overrides in `impl S3 for FS`, found exactly once across
`rustfs/src/storage/ecfs.rs` and `rustfs/src/storage/legacy_edge/`
(contract tests: `test_api_coverage.py`, run by `make script-tests`):
```bash
python3 scripts/s3-tests/api_coverage.py # summary + missing ops
+27 -10
View File
@@ -17,7 +17,8 @@
Compares the operations defined by the s3s `S3` trait (at the revision pinned
in the workspace Cargo.toml) against the methods RustFS overrides in
`impl S3 for FS` (rustfs/src/storage/ecfs.rs). Trait methods that are not
`impl S3 for FS` (rustfs/src/storage/ecfs.rs, or the rustfs/src/storage/legacy_edge/
directory a mechanical split of that file moves code into). Trait methods that are not
overridden fall back to the s3s default implementation, which returns
NotImplemented — so "overridden" is a faithful proxy for "implemented".
@@ -36,6 +37,8 @@ import sys
import tempfile
ECFS_PATH = "rustfs/src/storage/ecfs.rs"
ECFS_SPLIT_DIR = "rustfs/src/storage/legacy_edge"
IMPL_MARKER = "impl S3 for FS {"
TRAIT_RELPATH = "crates/s3s/src/s3_trait.rs"
@@ -97,14 +100,28 @@ def trait_operations(trait_source: pathlib.Path) -> dict[str, str]:
return ops
def implemented_operations(ecfs_source: pathlib.Path) -> set[str]:
src = ecfs_source.read_text(encoding="utf-8")
marker = "impl S3 for FS {"
start = src.find(marker)
if start < 0:
raise RuntimeError(f"`{marker}` not found in {ecfs_source}")
block = extract_block(src, start)
return set(re.findall(r"async fn (\w+)", block))
def ecfs_sources(root: pathlib.Path) -> list[pathlib.Path]:
"""Return ecfs.rs (required) followed by every .rs file under the split directory, in path order."""
required = root / ECFS_PATH
if not required.is_file():
raise RuntimeError(f"{required} is missing")
split_dir = root / ECFS_SPLIT_DIR
if not split_dir.exists():
return [required]
if not split_dir.is_dir():
raise RuntimeError(f"{split_dir} is not a directory")
return [required, *sorted(split_dir.rglob("*.rs"), key=lambda path: path.relative_to(root).as_posix())]
def implemented_operations(root: pathlib.Path) -> set[str]:
"""Return the methods of the one `impl S3 for FS` block across the ECFS source set."""
blocks = []
for source in ecfs_sources(root):
src = source.read_text(encoding="utf-8")
blocks += [extract_block(src, match.start()) for match in re.finditer(re.escape(IMPL_MARKER), src)]
if len(blocks) != 1:
raise RuntimeError(f"`{IMPL_MARKER}` must occur exactly once under {ECFS_PATH} and {ECFS_SPLIT_DIR}, found {len(blocks)}")
return set(re.findall(r"async fn (\w+)", blocks[0]))
def main() -> int:
@@ -122,7 +139,7 @@ def main() -> int:
trait_source = args.s3s_trait or locate_trait_source(git_url, rev)
all_ops = trait_operations(trait_source)
impl_ops = implemented_operations(root / ECFS_PATH)
impl_ops = implemented_operations(root)
stale = sorted(impl_ops - all_ops.keys())
implemented = sorted(op for op in all_ops if op in impl_ops)
+93
View File
@@ -0,0 +1,93 @@
#!/usr/bin/env python3
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contract tests for how api_coverage.py finds `impl S3 for FS` across the ECFS source set."""
from __future__ import annotations
import sys
import tempfile
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import api_coverage # noqa: E402
IMPL = "impl S3 for FS {\n async fn get_object(&self) {}\n async fn put_object(&self) {}\n}\n"
class ImplementedOperationsTests(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.root = Path(self._tmp.name)
def tearDown(self) -> None:
self._tmp.cleanup()
def write(self, relative: str, text: str) -> None:
path = self.root / relative
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
def test_reads_the_impl_from_ecfs_when_the_split_directory_is_absent(self) -> None:
self.write(api_coverage.ECFS_PATH, "struct FS;\n" + IMPL)
self.assertEqual(api_coverage.implemented_operations(self.root), {"get_object", "put_object"})
def test_finds_the_impl_after_it_moves_into_the_split_directory(self) -> None:
self.write(api_coverage.ECFS_PATH, "struct FS;\n")
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/s3_impl.rs", IMPL)
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/bucket_core.rs", "impl FS {\n async fn helper(&self) {}\n}\n")
self.assertEqual(api_coverage.implemented_operations(self.root), {"get_object", "put_object"})
def test_missing_ecfs_fails_even_when_the_split_directory_holds_the_impl(self) -> None:
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/s3_impl.rs", IMPL)
with self.assertRaisesRegex(RuntimeError, "ecfs.rs is missing"):
api_coverage.implemented_operations(self.root)
def test_an_impl_in_both_ecfs_and_the_split_directory_fails(self) -> None:
self.write(api_coverage.ECFS_PATH, IMPL)
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/s3_impl.rs", IMPL)
with self.assertRaisesRegex(RuntimeError, "exactly once.*found 2"):
api_coverage.implemented_operations(self.root)
def test_no_impl_anywhere_fails(self) -> None:
self.write(api_coverage.ECFS_PATH, "struct FS;\n")
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/bucket_core.rs", "impl FS {}\n")
with self.assertRaisesRegex(RuntimeError, "found 0"):
api_coverage.implemented_operations(self.root)
def test_an_impl_in_a_non_rust_file_of_the_split_directory_is_not_source(self) -> None:
self.write(api_coverage.ECFS_PATH, "struct FS;\n")
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/notes.md", IMPL)
with self.assertRaisesRegex(RuntimeError, "found 0"):
api_coverage.implemented_operations(self.root)
def test_a_split_path_that_is_a_file_fails(self) -> None:
self.write(api_coverage.ECFS_PATH, IMPL)
self.write(api_coverage.ECFS_SPLIT_DIR, "stray\n")
with self.assertRaisesRegex(RuntimeError, "not a directory"):
api_coverage.implemented_operations(self.root)
if __name__ == "__main__":
unittest.main()
+153
View File
@@ -0,0 +1,153 @@
#!/usr/bin/env bash
# Contract tests for scripts/guard_source_set.sh, the source-set helper that
# lets text guards read a file plus the directory a mechanical split moves its
# code into (rustfs/backlog#2765).
#
# Each case builds a throwaway tree; nothing in the real workspace is read.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
# shellcheck source=scripts/guard_source_set.sh
source "${GUARD_SOURCE_SET_SCRIPT:-$SCRIPT_DIR/guard_source_set.sh}"
TMP_ROOT="$(mktemp -d)"
trap 'chmod -R u+rwx "$TMP_ROOT" 2>/dev/null || true; rm -rf "$TMP_ROOT"' EXIT
passed=0
skipped=0
case_no=0
FX=""
STATUS=0
ERR=""
fail() {
echo "FAIL: $1" >&2
exit 1
}
new_case() {
case_no=$((case_no + 1))
FX="$TMP_ROOT/case$case_no"
mkdir -p "$FX"
}
put() {
mkdir -p "$(dirname "$FX/$1")"
printf 'fn guarded() {}\n' >"$FX/$1"
}
# run_set <args...>: call guard_source_set on the case tree, capturing status and stderr.
run_set() {
local err_file="$FX/.stderr"
STATUS=0
guard_source_set "$FX" "$@" 2>"$err_file" || STATUS=$?
ERR="$(cat "$err_file")"
}
sources_are() {
local expected actual
expected="$(printf '%s\n' "$@")"
actual="$(printf '%s\n' "${GUARD_SOURCES[@]}")"
[[ "$actual" == "$expected" ]] || fail "case $case_no: sources were [$actual], expected [$expected]"
}
ok() {
passed=$((passed + 1))
}
# --- positive cases ---------------------------------------------------------
# An absent split directory contributes nothing.
new_case
put src/usecase.rs
run_set src/usecase.rs src/usecase
[[ "$STATUS" == 0 ]] || fail "absent split dir: status $STATUS ($ERR)"
sources_are src/usecase.rs
ok
# A present split directory adds its .rs files, recursively, in byte order,
# after the required file; non-.rs files are not source.
new_case
put src/usecase.rs
put src/usecase/zeta.rs
put src/usecase/B.rs
put src/usecase/a.rs
put src/usecase/tests/nested.rs
printf 'not rust\n' >"$FX/src/usecase/notes.md"
run_set src/usecase.rs src/usecase
[[ "$STATUS" == 0 ]] || fail "present split dir: status $STATUS ($ERR)"
sources_are src/usecase.rs src/usecase/B.rs src/usecase/a.rs src/usecase/tests/nested.rs src/usecase/zeta.rs
ok
# Several split directories are read in argument order; an empty one adds nothing.
new_case
put src/ecfs.rs
put src/legacy_edge/b.rs
mkdir -p "$FX/src/empty"
put src/other/a.rs
run_set src/ecfs.rs src/legacy_edge src/empty src/missing src/other
[[ "$STATUS" == 0 ]] || fail "several split dirs: status $STATUS ($ERR)"
sources_are src/ecfs.rs src/legacy_edge/b.rs src/other/a.rs
ok
# --- negative cases ---------------------------------------------------------
# A missing required file fails and names the file.
new_case
run_set src/usecase.rs src/usecase
[[ "$STATUS" == 1 ]] || fail "missing required file: status $STATUS"
[[ "$ERR" == *"src/usecase.rs"* ]] || fail "missing required file: stderr does not name it: $ERR"
sources_are src/usecase.rs
ok
# A missing required file fails even when the split directory holds the code:
# the split must keep the file, and a guard must not pass on half its input.
new_case
put src/usecase/moved.rs
run_set src/usecase.rs src/usecase
[[ "$STATUS" == 1 ]] || fail "missing required file with split dir: status $STATUS"
[[ "$ERR" == *"is missing"* ]] || fail "missing required file with split dir: $ERR"
ok
# A required path that is a directory is not a source file.
new_case
mkdir -p "$FX/src/usecase.rs"
run_set src/usecase.rs src/usecase
[[ "$STATUS" == 1 ]] || fail "required directory: status $STATUS"
ok
# A split path that exists as a regular file fails rather than being skipped.
new_case
put src/usecase.rs
printf 'stray\n' >"$FX/src/usecase"
run_set src/usecase.rs src/usecase
[[ "$STATUS" == 1 ]] || fail "split path is a file: status $STATUS"
[[ "$ERR" == *"not a directory"* ]] || fail "split path is a file: $ERR"
ok
# A later split path that is a file still fails after an earlier valid one.
new_case
put src/usecase.rs
put src/usecase/a.rs
printf 'stray\n' >"$FX/src/second"
run_set src/usecase.rs src/usecase src/second
[[ "$STATUS" == 1 ]] || fail "second split path is a file: status $STATUS"
ok
# An unlistable split directory fails rather than contributing nothing.
new_case
put src/usecase.rs
put src/usecase/hidden/a.rs
chmod 000 "$FX/src/usecase/hidden"
if [[ "$(id -u)" == 0 ]]; then
echo "SKIP: unlistable split directory case needs a non-root user (root ignores directory permissions)"
skipped=$((skipped + 1))
else
run_set src/usecase.rs src/usecase
[[ "$STATUS" == 1 ]] || fail "unlistable split dir: status $STATUS"
[[ "$ERR" == *"cannot be listed"* ]] || fail "unlistable split dir: $ERR"
ok
fi
chmod 755 "$FX/src/usecase/hidden"
echo "guard_source_set contract tests: $passed passed, $skipped skipped"