mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-11 12:43:54 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4399c7e194 | |||
| c5885b328d | |||
| 682be074cd | |||
| 3e8f7a001f | |||
| 8172df44ec | |||
| 7b3701ddc6 | |||
| 2df55c6dea |
@@ -70,11 +70,6 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
||||
@echo "📦 Checking s3s footprint ratchet..."
|
||||
./scripts/check_s3s_footprint.sh
|
||||
|
||||
.PHONY: gateway-stack-release-boundary-check
|
||||
gateway-stack-release-boundary-check: ## Check no release build can enable the gateway-stack feature
|
||||
@echo "🚧 Checking gateway-stack release boundary..."
|
||||
./scripts/check_gateway_stack_release_boundary.sh
|
||||
|
||||
.PHONY: test-count-ratchet-check
|
||||
test-count-ratchet-check: ## Check no workspace crate fell below its test floor or rose above its #[ignore] ceiling in the committed baseline
|
||||
@echo "🧪 Checking per-crate test-count ratchet..."
|
||||
|
||||
@@ -20,13 +20,13 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check gateway-stack-release-boundary-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check gateway-stack-release-boundary-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check gateway-stack-release-boundary-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
@echo "✅ Fast development checks passed!"
|
||||
|
||||
@@ -61,8 +61,8 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_check_doc_paths.py
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
./scripts/check_s3s_footprint.sh --self-test
|
||||
./scripts/check_gateway_stack_release_boundary.sh --self-test
|
||||
./scripts/test_check_test_count_ratchet.sh
|
||||
./scripts/test_guard_source_set.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
@@ -94,6 +94,7 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_api_coverage.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_reference_parity.py
|
||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
|
||||
@@ -137,7 +137,7 @@ filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
# These handler/store fixtures and the relocated-pool GET compose deep storage
|
||||
@@ -455,7 +455,7 @@ filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
|
||||
@@ -13,5 +13,5 @@
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=65
|
||||
s3_error_lines=217
|
||||
s3_error_lines=219
|
||||
ecstore_files=0
|
||||
|
||||
@@ -25,7 +25,6 @@ rustfs-data-usage 58 0
|
||||
rustfs-ecstore 7858 2
|
||||
rustfs-extension-schema 23 0
|
||||
rustfs-filemeta 361 0
|
||||
rustfs-gateway-adapter 36 0
|
||||
rustfs-heal 1128 3
|
||||
rustfs-heal-contracts 8 0
|
||||
rustfs-iam 415 0
|
||||
|
||||
@@ -97,10 +97,6 @@ runs:
|
||||
shell: bash
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check gateway-stack release boundary
|
||||
shell: bash
|
||||
run: ./scripts/check_gateway_stack_release_boundary.sh
|
||||
|
||||
- name: Check per-crate test-count ratchet
|
||||
shell: bash
|
||||
run: ./scripts/check_test_count_ratchet.sh
|
||||
|
||||
@@ -115,7 +115,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=4$' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -705,53 +705,6 @@ jobs:
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
test-and-lint-gateway-stack:
|
||||
name: Test and Lint (gateway-stack)
|
||||
# The gateway S3 stack (rustfs/backlog#2734) sits behind the rustfs
|
||||
# `gateway-stack` feature, which no other pull-request lane lints or tests.
|
||||
# Unlike the dormant feature lanes above, this one runs for every code
|
||||
# change: each operation slice puts its handler tests, its authorization
|
||||
# census and its end-to-end refusal test behind the feature, and a lane that
|
||||
# ran only on demand would let them rot behind a green pull request.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# Self-hosted for build scale and cache ownership: a workspace-scale Cargo
|
||||
# build that reads the ci-dev cache written on this image.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
# A feature that does not compile fails here; nothing below is skipped
|
||||
# into a pass.
|
||||
- name: Run clippy with gateway-stack
|
||||
run: cargo clippy --locked -p rustfs --all-targets --features gateway-stack -- -D warnings
|
||||
|
||||
# The gateway stack's own tests: the assembly, the authorizer table and
|
||||
# its census, the handler template, the migrated use cases, and the
|
||||
# embedded server driven through the whole middleware stack.
|
||||
- name: Run tests with gateway-stack
|
||||
env:
|
||||
# The same bounded link concurrency as the other feature lanes.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
cargo nextest run --profile ci --locked -p rustfs --features gateway-stack --no-tests fail \
|
||||
--status-level all --final-status-level all \
|
||||
-E 'test(/gateway_stack|gateway_handlers|server::stack::|list_buckets|legacy_stack_is_always_selectable/) | binary(embedded_gateway_stack_test)'
|
||||
|
||||
build-rustfs-debug-binary:
|
||||
name: Build RustFS Debug Binary
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
@@ -1626,7 +1579,6 @@ jobs:
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- uring-integration
|
||||
- e2e-tests
|
||||
@@ -1661,7 +1613,6 @@ jobs:
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- build-rustfs-debug-binary-rio-v2
|
||||
- uring-integration
|
||||
|
||||
Generated
+2
-140
@@ -4698,12 +4698,6 @@ version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
|
||||
|
||||
[[package]]
|
||||
name = "futures-timer"
|
||||
version = "3.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.34"
|
||||
@@ -7920,34 +7914,10 @@ version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared 0.14.0",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"phf_shared 0.14.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared 0.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.12.1"
|
||||
@@ -9639,8 +9609,6 @@ dependencies = [
|
||||
"rustfs-ecstore",
|
||||
"rustfs-extension-schema",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-gateway",
|
||||
"rustfs-gateway-adapter",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-heal",
|
||||
"rustfs-heal-contracts",
|
||||
@@ -10031,112 +9999,6 @@ dependencies = [
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway"
|
||||
version = "0.57.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"futures-timer",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"httparse",
|
||||
"hyper",
|
||||
"pin-project-lite",
|
||||
"rustfs-gateway-core",
|
||||
"rustfs-gateway-http",
|
||||
"rustfs-gateway-macros",
|
||||
"rustfs-gateway-sig",
|
||||
"rustfs-gateway-stream",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-gateway-xml",
|
||||
"sha2 0.11.1",
|
||||
"tokio",
|
||||
"tower",
|
||||
"tracing",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-adapter"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"rustfs-credentials",
|
||||
"rustfs-gateway",
|
||||
"rustfs-iam",
|
||||
"rustfs-policy",
|
||||
"serde_json",
|
||||
"time",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-core"
|
||||
version = "0.54.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http 1.5.0",
|
||||
"md-5 0.11.0",
|
||||
"percent-encoding",
|
||||
"rustfs-gateway-http",
|
||||
"rustfs-gateway-sig",
|
||||
"rustfs-gateway-stream",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-gateway-xml",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-http"
|
||||
version = "0.1.4"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
"httparse",
|
||||
"memchr",
|
||||
"phf 0.14.0",
|
||||
"rustfs-gateway-stream",
|
||||
"rustfs-gateway-types",
|
||||
"sha2 0.11.1",
|
||||
"smallvec",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-macros"
|
||||
version = "0.1.4"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-sig"
|
||||
version = "0.13.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
"rustfs-gateway-http",
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.1",
|
||||
"smallvec",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-stream"
|
||||
version = "0.2.0"
|
||||
@@ -12741,10 +12603,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -26,7 +26,6 @@ members = [
|
||||
"crates/ecstore", # Erasure coding storage implementation
|
||||
"crates/e2e_test", # End-to-end test suite
|
||||
"crates/filemeta", # File metadata management
|
||||
"crates/gateway-adapter", # RustFS implementations of the gateway extension traits
|
||||
"crates/heal", # Erasure set and object healing
|
||||
"crates/heal-contracts", # Heal request/response channel contracts
|
||||
"crates/iam", # Identity and Access Management
|
||||
@@ -94,7 +93,6 @@ redundant_clone = "warn"
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.5" }
|
||||
rustfs-gateway-adapter = { path = "crates/gateway-adapter", version = "1.0.5" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.5" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.5" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.5" }
|
||||
@@ -151,10 +149,6 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.32.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.57.0", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "rustfs-gateway-adapter"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "RustFS implementations of the rustfs-gateway extension traits (credentials and policy facts over IAM)."
|
||||
keywords = ["rustfs", "gateway", "s3", "iam"]
|
||||
categories = ["web-programming", "authentication"]
|
||||
authors.workspace = true
|
||||
publish = false
|
||||
|
||||
[package.metadata.gateway]
|
||||
# Documentary only: this crate is a ring-2 consumer of the gateway (it depends on
|
||||
# RustFS business crates). The RustFS repository does not run the gateway's ring guard.
|
||||
ring = 2
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
rustfs-gateway = { workspace = true }
|
||||
rustfs-credentials = { workspace = true }
|
||||
rustfs-iam = { workspace = true }
|
||||
rustfs-policy = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
time = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt"] }
|
||||
@@ -1,446 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway's [`CredentialProvider`], answered by RustFS IAM.
|
||||
//!
|
||||
//! Responsible for: resolving one access key id to the secret the gateway verifies a signature
|
||||
//! with, and classifying every other outcome as either a business result
|
||||
//! ([`CredentialLookup::NotFound`], eligible for the gateway's negative cache) or an operational
|
||||
//! fault ([`ProviderError`], counted and never cached).
|
||||
//! NOT responsible for: authorization of any kind, session-token claim validation, or deciding
|
||||
//! what the client is answered with. The gateway answers every unusable credential identically.
|
||||
//!
|
||||
//! # Secret handling
|
||||
//!
|
||||
//! Secrets travel only inside the gateway's [`Credentials`], whose `Debug` redacts them. No type
|
||||
//! in this module derives or implements `Debug`, and no error produced here carries text: the
|
||||
//! IAM error is classified and dropped. A secret that fails the crate's effective-length rule
|
||||
//! (`crate::MIN_SECRET_KEY_LEN` bytes once trailing NULs are removed) is never handed to the
|
||||
//! gateway, whatever the lookup says about its validity.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::{BoxFuture, CredentialLookup, CredentialProvider, Credentials, ProviderError, SessionBinding};
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::store::Store;
|
||||
use rustfs_iam::sys::IamSys;
|
||||
use rustfs_policy::auth::{ACCOUNT_OFF, UserIdentity};
|
||||
|
||||
use crate::root::{RootSource, is_usable_secret};
|
||||
|
||||
/// The opaque issuer recorded in the [`rustfs_gateway::SessionBinding`] of every RustFS STS
|
||||
/// credential.
|
||||
pub const STS_SESSION_ISSUER: &str = "rustfs-sts";
|
||||
|
||||
/// Access keys with this prefix belong to Keystone token authentication, which never verifies an
|
||||
/// AWS signature.
|
||||
const KEYSTONE_ACCESS_KEY_PREFIX: &str = "keystone:";
|
||||
|
||||
/// The narrow slice of IAM that a credential lookup needs.
|
||||
///
|
||||
/// Exists so the adapter can be tested against a fake: IAM exports no in-memory backend.
|
||||
/// Implemented for [`IamSys`], whose `check_key` loads a user from storage on a cache miss.
|
||||
pub trait IamKeyLookup: Send + Sync + 'static {
|
||||
/// Whether the IAM store has finished loading.
|
||||
fn is_ready(&self) -> bool;
|
||||
|
||||
/// Resolves an access key, with [`IamSys::check_key`]'s contract: `Ok((None, _))` for an
|
||||
/// unknown key, `Ok((Some(identity), valid))` otherwise, where `valid` is false for a
|
||||
/// disabled, expired, or malformed credential.
|
||||
fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>>;
|
||||
}
|
||||
|
||||
impl<T: Store> IamKeyLookup for IamSys<T> {
|
||||
fn is_ready(&self) -> bool {
|
||||
IamSys::is_ready(self)
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
|
||||
Box::pin(IamSys::check_key(self, access_key))
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`CredentialProvider`] over RustFS IAM and the live root credentials.
|
||||
///
|
||||
/// Built only by [`crate::IamAdapters::new`], so it always reads the same [`RootSource`] as the
|
||||
/// policy source it was assembled with.
|
||||
pub struct IamCredentials {
|
||||
iam: Arc<dyn IamKeyLookup>,
|
||||
root: Arc<dyn RootSource>,
|
||||
}
|
||||
|
||||
impl IamCredentials {
|
||||
pub(crate) fn new(iam: Arc<dyn IamKeyLookup>, root: Arc<dyn RootSource>) -> Self {
|
||||
Self { iam, root }
|
||||
}
|
||||
|
||||
async fn resolve(&self, access_key: &str) -> Result<CredentialLookup, ProviderError> {
|
||||
// Deliberate change from the legacy s3s path (`rustfs/src/auth.rs` `get_secret_key`),
|
||||
// which answered a `keystone:` key with an empty secret. Keystone authenticates by token,
|
||||
// so there is no AWS secret to verify a signature against: the key is unknown here.
|
||||
if access_key.starts_with(KEYSTONE_ACCESS_KEY_PREFIX) {
|
||||
return Ok(CredentialLookup::NotFound);
|
||||
}
|
||||
|
||||
// Legacy answers every key with an internal error while the root credentials are not
|
||||
// initialized; without them no key can be classified as root or not.
|
||||
let Some(root) = self.root.current() else {
|
||||
return Err(ProviderError::Unavailable);
|
||||
};
|
||||
// Byte-exact, as the trait requires: the access key id is signed material.
|
||||
if root.access_key() == access_key {
|
||||
return Ok(CredentialLookup::Found(root.credentials().clone_credentials()));
|
||||
}
|
||||
|
||||
if !self.iam.is_ready() {
|
||||
return Err(ProviderError::Unavailable);
|
||||
}
|
||||
|
||||
match self.iam.check_key(access_key).await {
|
||||
// IAM refuses such a secret through `valid`, but this provider does not rely on every
|
||||
// lookup doing so: a short secret is never usable, so it is not a credential at all.
|
||||
Ok((Some(identity), _)) if !is_usable_secret(identity.credentials.secret_key.as_bytes()) => {
|
||||
Ok(CredentialLookup::NotFound)
|
||||
}
|
||||
Ok((Some(identity), valid)) => to_gateway_credentials(access_key, &identity, valid).map(CredentialLookup::Found),
|
||||
Ok((None, _)) => Ok(CredentialLookup::NotFound),
|
||||
Err(error) => classify_iam_error(&error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Classifies an IAM failure without keeping any of its text.
|
||||
fn classify_iam_error(error: &IamError) -> Result<CredentialLookup, ProviderError> {
|
||||
match error {
|
||||
IamError::NoSuchUser(_)
|
||||
| IamError::NoSuchServiceAccount(_)
|
||||
| IamError::NoSuchTempAccount(_)
|
||||
| IamError::NoSuchAccount(_) => Ok(CredentialLookup::NotFound),
|
||||
IamError::IamSysNotInitialized => Err(ProviderError::Unavailable),
|
||||
_ => Err(ProviderError::Backend),
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the gateway credential for one IAM identity. No authorization happens here: a
|
||||
/// disabled or invalid principal still resolves, marked disabled, and the gateway refuses it
|
||||
/// exactly as it refuses an unknown key.
|
||||
fn to_gateway_credentials(access_key: &str, identity: &UserIdentity, valid: bool) -> Result<Credentials, ProviderError> {
|
||||
let cred = &identity.credentials;
|
||||
let mut credentials = Credentials::new(access_key, cred.secret_key.as_bytes()).map_err(|_| ProviderError::Backend)?;
|
||||
|
||||
// A service account carries a claims token internally, but its clients never present it, so
|
||||
// it authenticates as a long-term credential (as `check_session_token` treats it in legacy).
|
||||
if !cred.session_token.is_empty() && !cred.is_service_account() {
|
||||
// A temporary credential without an expiry cannot become a gateway session, which always
|
||||
// ends. Refusing it as a fault is the fail-closed answer; a never-ending session is not.
|
||||
let expiration = cred.expiration.ok_or(ProviderError::Backend)?;
|
||||
let binding = SessionBinding::new(STS_SESSION_ISSUER, expiration.unix_timestamp()).map_err(|_| ProviderError::Backend)?;
|
||||
credentials = credentials
|
||||
.with_session(&cred.session_token, binding)
|
||||
.map_err(|_| ProviderError::Backend)?;
|
||||
}
|
||||
|
||||
if !valid || cred.status == ACCOUNT_OFF {
|
||||
credentials = credentials.disable();
|
||||
}
|
||||
Ok(credentials)
|
||||
}
|
||||
|
||||
impl CredentialProvider for IamCredentials {
|
||||
fn lookup<'a>(&'a self, access_key_id: &'a str) -> BoxFuture<'a, Result<CredentialLookup, ProviderError>> {
|
||||
Box::pin(self.resolve(access_key_id))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::root::{RootCell, RootCredentials};
|
||||
use rustfs_credentials::Credentials as IamCredential;
|
||||
use rustfs_gateway::RequestNow;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
const ACCESS_KEY: &str = "AKIDUSER0001";
|
||||
const SECRET: &str = "user-secret-key-0001";
|
||||
|
||||
type Answer = Box<dyn Fn() -> Result<(Option<UserIdentity>, bool), IamError> + Send + Sync>;
|
||||
|
||||
struct FakeIam {
|
||||
ready: bool,
|
||||
answer: Answer,
|
||||
calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl IamKeyLookup for FakeIam {
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
let answer = (self.answer)();
|
||||
Box::pin(async move { answer })
|
||||
}
|
||||
}
|
||||
|
||||
fn fake(
|
||||
ready: bool,
|
||||
answer: impl Fn() -> Result<(Option<UserIdentity>, bool), IamError> + Send + Sync + 'static,
|
||||
) -> Arc<FakeIam> {
|
||||
Arc::new(FakeIam {
|
||||
ready,
|
||||
answer: Box::new(answer),
|
||||
calls: AtomicUsize::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
fn user(status: &str) -> IamCredential {
|
||||
IamCredential {
|
||||
access_key: ACCESS_KEY.to_owned(),
|
||||
secret_key: SECRET.to_owned(),
|
||||
status: status.to_owned(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn found(cred: IamCredential, valid: bool) -> Result<(Option<UserIdentity>, bool), IamError> {
|
||||
Ok((Some(UserIdentity::new(cred)), valid))
|
||||
}
|
||||
|
||||
fn root_cell() -> Arc<RootCell> {
|
||||
let root = RootCredentials::new("rootadmin", "root-secret-key", "sts-signing-key").expect("valid root credentials");
|
||||
Arc::new(RootCell::new(Some(root)))
|
||||
}
|
||||
|
||||
async fn lookup(iam: Arc<FakeIam>, access_key: &str) -> Result<CredentialLookup, ProviderError> {
|
||||
IamCredentials::new(iam, root_cell()).lookup(access_key).await
|
||||
}
|
||||
|
||||
fn expect_found(result: Result<CredentialLookup, ProviderError>) -> Credentials {
|
||||
match result {
|
||||
Ok(CredentialLookup::Found(credentials)) => credentials,
|
||||
Ok(CredentialLookup::NotFound) => panic!("expected Found, got NotFound"),
|
||||
Err(error) => panic!("expected Found, got {error:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_not_found(result: &Result<CredentialLookup, ProviderError>) -> bool {
|
||||
matches!(result, Ok(CredentialLookup::NotFound))
|
||||
}
|
||||
|
||||
/// Positive — an enabled user resolves to exactly its secret, as a long-term credential.
|
||||
#[tokio::test]
|
||||
async fn credentials_found_user_resolves_to_its_secret() {
|
||||
let iam = fake(true, || found(user("on"), true));
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
assert_eq!(credentials.secret().expose(), SECRET.as_bytes());
|
||||
assert_eq!(credentials.identity().access_key_id(), ACCESS_KEY);
|
||||
assert!(!credentials.is_disabled());
|
||||
assert!(credentials.session_binding().is_none());
|
||||
}
|
||||
|
||||
/// Negative — a key IAM does not know is a business result, not a fault.
|
||||
#[tokio::test]
|
||||
async fn credentials_unknown_key_is_not_found() {
|
||||
let iam = fake(true, || Ok((None, false)));
|
||||
assert!(is_not_found(&lookup(iam, "AKIDUNKNOWN").await));
|
||||
}
|
||||
|
||||
/// Negative — every "no such principal" error IAM can surface is `NotFound`. Answering these
|
||||
/// as `Backend` would keep them out of the negative cache and count every guessed key as a
|
||||
/// backend fault.
|
||||
#[tokio::test]
|
||||
async fn credentials_no_such_principal_errors_are_not_found() {
|
||||
let errors: [fn() -> IamError; 4] = [
|
||||
|| IamError::NoSuchUser(ACCESS_KEY.to_owned()),
|
||||
|| IamError::NoSuchServiceAccount(ACCESS_KEY.to_owned()),
|
||||
|| IamError::NoSuchTempAccount(ACCESS_KEY.to_owned()),
|
||||
|| IamError::NoSuchAccount(ACCESS_KEY.to_owned()),
|
||||
];
|
||||
for error in errors {
|
||||
let iam = fake(true, move || Err(error()));
|
||||
let result = lookup(iam, ACCESS_KEY).await;
|
||||
assert!(is_not_found(&result), "{:?} must be NotFound", error());
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — any other IAM failure is a backend fault, never `NotFound` and never `Found`.
|
||||
#[tokio::test]
|
||||
async fn credentials_backend_error_is_backend() {
|
||||
let iam = fake(true, || Err(IamError::StringError("storage read failed".to_owned())));
|
||||
assert!(matches!(lookup(iam, ACCESS_KEY).await, Err(ProviderError::Backend)));
|
||||
}
|
||||
|
||||
/// Negative — a `keystone:` key never reaches IAM and never resolves to a secret.
|
||||
///
|
||||
/// Deliberate change from the legacy s3s path, which answered such a key with an empty
|
||||
/// secret: Keystone authenticates by token, so there is no AWS secret to verify against.
|
||||
#[tokio::test]
|
||||
async fn credentials_keystone_prefix_is_not_found_without_iam() {
|
||||
let iam = fake(true, || found(user("on"), true));
|
||||
let result = lookup(iam.clone(), "keystone:project-user").await;
|
||||
assert!(is_not_found(&result));
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// Negative — a switched-off or otherwise invalid user resolves, but disabled, and the same
|
||||
/// user enabled does not.
|
||||
#[tokio::test]
|
||||
async fn credentials_disabled_user_is_disabled() {
|
||||
let off = expect_found(lookup(fake(true, || found(user("off"), false)), ACCESS_KEY).await);
|
||||
assert!(off.is_disabled());
|
||||
assert!(off.admit(None, RequestNow::from_unix_seconds(0)).is_err());
|
||||
|
||||
let invalid = expect_found(lookup(fake(true, || found(user("on"), false)), ACCESS_KEY).await);
|
||||
assert!(invalid.is_disabled());
|
||||
|
||||
let on = expect_found(lookup(fake(true, || found(user("on"), true)), ACCESS_KEY).await);
|
||||
assert!(!on.is_disabled());
|
||||
}
|
||||
|
||||
/// Positive — an STS credential carries its token and its lifetime as one session binding.
|
||||
#[tokio::test]
|
||||
async fn credentials_sts_carries_session_binding() {
|
||||
let expires_at = 4_000_000_000_i64;
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user("on");
|
||||
cred.session_token = "sts-session-token".to_owned();
|
||||
cred.parent_user = "parent".to_owned();
|
||||
cred.expiration = Some(OffsetDateTime::from_unix_timestamp(expires_at).expect("valid timestamp"));
|
||||
found(cred, true)
|
||||
});
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
let binding = credentials.session_binding().expect("an STS credential has a session");
|
||||
assert_eq!(binding.expires_at_unix_seconds(), expires_at);
|
||||
assert_eq!(binding.issuer(), STS_SESSION_ISSUER);
|
||||
let now = RequestNow::from_unix_seconds(expires_at - 1);
|
||||
assert!(credentials.admit(Some(b"sts-session-token"), now).is_ok());
|
||||
assert!(credentials.admit(None, now).is_err());
|
||||
}
|
||||
|
||||
/// Negative — an STS credential with no expiry cannot be expressed as a session the gateway
|
||||
/// would ever expire, so it is a backend fault rather than a session that never ends.
|
||||
#[tokio::test]
|
||||
async fn credentials_sts_without_expiry_is_backend() {
|
||||
let iam = fake(true, || {
|
||||
let mut cred = user("on");
|
||||
cred.session_token = "sts-session-token".to_owned();
|
||||
found(cred, true)
|
||||
});
|
||||
assert!(matches!(lookup(iam, ACCESS_KEY).await, Err(ProviderError::Backend)));
|
||||
}
|
||||
|
||||
/// Positive — a service account carries a token internally but authenticates as a long-term
|
||||
/// credential: clients never present that token.
|
||||
#[tokio::test]
|
||||
async fn credentials_service_account_is_long_term() {
|
||||
let iam = fake(true, || {
|
||||
let mut cred = user("on");
|
||||
cred.session_token = "service-account-claims-token".to_owned();
|
||||
cred.parent_user = "parent".to_owned();
|
||||
cred.claims = Some([(rustfs_credentials::IAM_POLICY_CLAIM_NAME_SA.to_owned(), "inherited-policy".into())].into());
|
||||
found(cred, true)
|
||||
});
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
assert!(credentials.session_binding().is_none());
|
||||
assert!(credentials.admit(None, RequestNow::from_unix_seconds(0)).is_ok());
|
||||
}
|
||||
|
||||
/// Negative — before IAM is ready a lookup is `Unavailable`, not `NotFound`: a key must not
|
||||
/// be negatively cached because the store had not loaded yet.
|
||||
#[tokio::test]
|
||||
async fn credentials_iam_not_ready_is_unavailable() {
|
||||
let iam = fake(false, || found(user("on"), true));
|
||||
assert!(matches!(lookup(iam.clone(), ACCESS_KEY).await, Err(ProviderError::Unavailable)));
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
let uninitialized = fake(true, || Err(IamError::IamSysNotInitialized));
|
||||
assert!(matches!(lookup(uninitialized, ACCESS_KEY).await, Err(ProviderError::Unavailable)));
|
||||
}
|
||||
|
||||
/// Positive — the injected root credentials resolve without IAM, even before it is ready, and
|
||||
/// nothing else is answered with the root secret.
|
||||
#[tokio::test]
|
||||
async fn credentials_root_resolves_without_iam() {
|
||||
let iam = fake(false, || found(user("on"), true));
|
||||
let provider = IamCredentials::new(iam.clone(), root_cell());
|
||||
let root = expect_found(provider.lookup("rootadmin").await);
|
||||
assert_eq!(root.secret().expose(), b"root-secret-key");
|
||||
assert!(matches!(provider.lookup("ROOTADMIN").await, Err(ProviderError::Unavailable)));
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// Negative — a lookup that calls an identity valid but holds an empty or short secret is not
|
||||
/// trusted: such a secret is not a credential, so the key is `NotFound`, never `Found`.
|
||||
#[tokio::test]
|
||||
async fn credentials_short_secret_is_never_found() {
|
||||
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user("on");
|
||||
cred.secret_key = weak.to_owned();
|
||||
found(cred, true)
|
||||
});
|
||||
let result = lookup(iam, ACCESS_KEY).await;
|
||||
assert!(is_not_found(&result), "{weak:?} must be NotFound");
|
||||
}
|
||||
for usable in ["12345678", "abc\0defg"] {
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user("on");
|
||||
cred.secret_key = usable.to_owned();
|
||||
found(cred, true)
|
||||
});
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
assert_eq!(credentials.secret().expose(), usable.as_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — a switched-off account is disabled even when the lookup calls it valid.
|
||||
#[tokio::test]
|
||||
async fn credentials_off_status_alone_disables() {
|
||||
let off = expect_found(lookup(fake(true, || found(user("off"), true)), ACCESS_KEY).await);
|
||||
assert!(off.is_disabled());
|
||||
assert!(off.admit(None, RequestNow::from_unix_seconds(0)).is_err());
|
||||
}
|
||||
|
||||
/// Negative — an STS credential admits only its own session's token: the token of another
|
||||
/// session, or none, is refused.
|
||||
#[tokio::test]
|
||||
async fn credentials_sts_refuses_another_sessions_token() {
|
||||
let expires_at = 4_000_000_000_i64;
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user("on");
|
||||
cred.session_token = "session-token-a".to_owned();
|
||||
cred.parent_user = "parent".to_owned();
|
||||
cred.expiration = Some(OffsetDateTime::from_unix_timestamp(expires_at).expect("valid timestamp"));
|
||||
found(cred, true)
|
||||
});
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
let now = RequestNow::from_unix_seconds(expires_at - 1);
|
||||
assert!(credentials.admit(Some(b"session-token-b"), now).is_err());
|
||||
assert!(credentials.admit(Some(b"session-token-a"), now).is_ok());
|
||||
}
|
||||
|
||||
/// Negative — without root credentials no key can be classified, so every lookup is
|
||||
/// `Unavailable` and IAM is not asked.
|
||||
#[tokio::test]
|
||||
async fn credentials_without_root_are_unavailable() {
|
||||
let iam = fake(true, || found(user("on"), true));
|
||||
let provider = IamCredentials::new(iam.clone(), Arc::new(RootCell::new(None)));
|
||||
assert!(matches!(provider.lookup(ACCESS_KEY).await, Err(ProviderError::Unavailable)));
|
||||
assert!(matches!(provider.lookup("rootadmin").await, Err(ProviderError::Unavailable)));
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! RustFS implementations of the `rustfs-gateway` extension traits.
|
||||
//!
|
||||
//! This is the gateway's ring 2: it may depend on RustFS business crates, and nothing in the
|
||||
//! gateway depends on it. Each module adapts one extension point to the RustFS subsystem that
|
||||
//! owns the answer.
|
||||
|
||||
pub mod credentials;
|
||||
pub mod policy;
|
||||
pub mod root;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::{Authenticator, CredentialProvider, ServiceBuilder};
|
||||
|
||||
pub use credentials::{IamCredentials, IamKeyLookup, STS_SESSION_ISSUER};
|
||||
pub use policy::{IamPolicySource, IamPrincipalLookup, PrincipalKind, RustfsPrincipal};
|
||||
pub use root::{MIN_SECRET_KEY_LEN, RootCredentials, RootCredentialsError, RootSource};
|
||||
|
||||
/// The IAM-backed credential provider and policy source, assembled together.
|
||||
///
|
||||
/// Both halves read the same IAM handle and the same live [`RootSource`], so a key cannot be root
|
||||
/// to one and an ordinary user to the other. Neither half can be taken out on its own: the fields
|
||||
/// are private, and [`IamAdapters::install`] consumes the pair and hands both to one builder.
|
||||
///
|
||||
/// Mixing the halves of two assemblies does not compile:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn mix(a: rustfs_gateway_adapter::IamAdapters, b: rustfs_gateway_adapter::IamAdapters) {
|
||||
/// let _mixed = (a.credentials, b.policy);
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// Nor does building a half outside the pair:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// use std::sync::Arc;
|
||||
/// use rustfs_gateway_adapter::{IamPolicySource, IamPrincipalLookup, RootSource};
|
||||
/// fn build(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) {
|
||||
/// let _alone = IamPolicySource::new(iam, root);
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// Installing the pair is the one supported path:
|
||||
///
|
||||
/// ```no_run
|
||||
/// use std::sync::Arc;
|
||||
/// use rustfs_gateway::{RegionSet, ServiceBuilder, SigV4Authenticator};
|
||||
/// use rustfs_gateway_adapter::{IamAdapters, IamPrincipalLookup, RootSource};
|
||||
/// fn assemble(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>, regions: RegionSet) -> ServiceBuilder {
|
||||
/// IamAdapters::new(iam, root).install(ServiceBuilder::new(), |provider| SigV4Authenticator::new(provider, regions))
|
||||
/// }
|
||||
/// ```
|
||||
pub struct IamAdapters {
|
||||
credentials: IamCredentials,
|
||||
policy: IamPolicySource,
|
||||
}
|
||||
|
||||
impl IamAdapters {
|
||||
/// Assembles both adapters over one IAM handle and one root source. The binary passes a
|
||||
/// source that reads its live root credentials on every call (see [`RootSource`]).
|
||||
#[must_use]
|
||||
pub fn new(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) -> Self {
|
||||
Self {
|
||||
credentials: IamCredentials::new(iam.clone(), root.clone()),
|
||||
policy: IamPolicySource::new(iam, root),
|
||||
}
|
||||
}
|
||||
|
||||
/// Installs both halves into `builder`: the credential provider through the authenticator
|
||||
/// `authenticator` builds over it (for example `SigV4Authenticator::new(provider, regions)`
|
||||
/// with the RustFS profile applied), and the policy source.
|
||||
///
|
||||
/// Consumes the pair and gives both halves to this one builder. The `authenticator` closure
|
||||
/// receives the provider handle and may keep a clone of it; nothing here prevents that, so
|
||||
/// the closure must not hand it to a second builder.
|
||||
#[must_use]
|
||||
pub fn install<A: Authenticator>(
|
||||
self,
|
||||
builder: ServiceBuilder,
|
||||
authenticator: impl FnOnce(Arc<dyn CredentialProvider>) -> A,
|
||||
) -> ServiceBuilder {
|
||||
let authenticator = authenticator(Arc::new(self.credentials));
|
||||
builder.authenticator(authenticator).policy_source(self.policy)
|
||||
}
|
||||
}
|
||||
@@ -1,779 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway's [`PolicySource`], answered by RustFS IAM.
|
||||
//!
|
||||
//! Responsible for: the one reading of the caller's IAM facts a request is judged against — which
|
||||
//! principal kind it is, the user it acts for, its groups, its token claims, and whether it owns
|
||||
//! the deployment — packed as a [`RustfsPrincipal`] into the gateway's [`PolicySnapshot`].
|
||||
//! NOT responsible for: evaluating any policy, choosing which policy documents apply (that stays
|
||||
//! in `IamSys::is_allowed`, which owns the STS, OIDC, session-policy and service-account rules), or
|
||||
//! verifying the signature or the session token (the gateway did both before calling this).
|
||||
//! Upstream: the gateway's verified [`Identity`]. Downstream: the RustFS authorizer, which reads
|
||||
//! the payload back with `PolicySnapshot::get::<RustfsPrincipal>()`.
|
||||
//!
|
||||
//! # Failure classes
|
||||
//!
|
||||
//! Every refusal is a [`PolicyError`] with a constant label and nothing from IAM or the request:
|
||||
//! IAM not ready, not initialized, or failing, or root credentials not initialized, is
|
||||
//! [`PolicyError::unavailable`]; a principal IAM no longer knows is [`UNKNOWN_PRINCIPAL`]; a
|
||||
//! disabled one, or one whose secret fails the effective-length rule, is
|
||||
//! [`DISABLED_PRINCIPAL`]; an identity
|
||||
//! whose credential kind disagrees with IAM's record is [`SESSION_MISMATCH`]; a token whose claims
|
||||
//! cannot be read is [`PolicyError::malformed`]. None of them is ever an empty snapshot: an empty
|
||||
//! snapshot is what an anonymous caller gets, and answering a fault with it would judge an
|
||||
//! authenticated caller as anonymous.
|
||||
//!
|
||||
//! # Secret handling
|
||||
//!
|
||||
//! [`RustfsPrincipal`] holds no secret key and no session token, and implements no `Debug`, so a
|
||||
//! snapshot formatted into a log cannot print its claims either. The root credentials and the STS
|
||||
//! token-signing key are read per request from the shared [`RootSource`] and never stored here.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_credentials::Credentials as IamCredential;
|
||||
use rustfs_gateway::{BoxFuture, Identity, PolicyError, PolicySnapshot, PolicySource};
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::store::Store;
|
||||
use rustfs_iam::sys::{
|
||||
IamSys, SESSION_POLICY_NAME, get_claims_from_token_with_secret, get_claims_from_token_with_secret_allow_missing_exp,
|
||||
is_rustfs_oidc_claims,
|
||||
};
|
||||
use rustfs_policy::auth::ACCOUNT_OFF;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::credentials::IamKeyLookup;
|
||||
use crate::root::{RootCredentials, RootSource, is_usable_secret};
|
||||
|
||||
/// [`PolicyError`] label: IAM does not know the authenticated principal (any longer).
|
||||
pub const UNKNOWN_PRINCIPAL: &str = "unknown-principal";
|
||||
/// [`PolicyError`] label: IAM holds the principal disabled, expired, or otherwise invalid.
|
||||
pub const DISABLED_PRINCIPAL: &str = "disabled-principal";
|
||||
/// [`PolicyError`] label: the identity's credential kind (session or long-term) disagrees with
|
||||
/// what IAM holds for its access key.
|
||||
pub const SESSION_MISMATCH: &str = "session-mismatch";
|
||||
|
||||
/// The narrow slice of IAM a policy reading needs, on top of [`IamKeyLookup`].
|
||||
///
|
||||
/// Implemented for [`IamSys`]; exists so the source can be tested against a fake.
|
||||
pub trait IamPrincipalLookup: IamKeyLookup {
|
||||
/// The built-in IAM groups `user` belongs to now, as `IamSys::user_group_memberships`
|
||||
/// answers. These may include groups granted after a credential was issued.
|
||||
fn group_memberships(&self, user: &str) -> Vec<String>;
|
||||
}
|
||||
|
||||
impl<T: Store> IamPrincipalLookup for IamSys<T> {
|
||||
fn group_memberships(&self, user: &str) -> Vec<String> {
|
||||
IamSys::user_group_memberships(self, user)
|
||||
}
|
||||
}
|
||||
|
||||
/// Which kind of credential the principal authenticated with.
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PrincipalKind {
|
||||
/// The deployment's root credentials.
|
||||
Root,
|
||||
/// A long-term IAM user.
|
||||
User,
|
||||
/// A service account, acting for its parent user.
|
||||
ServiceAccount,
|
||||
/// An STS temporary credential, acting for its parent user.
|
||||
Temporary,
|
||||
}
|
||||
|
||||
/// The caller's IAM facts: the payload of every [`PolicySnapshot`] [`IamPolicySource`] produces.
|
||||
///
|
||||
/// Carries no secret key and no session token, and deliberately implements no `Debug`.
|
||||
pub struct RustfsPrincipal {
|
||||
kind: PrincipalKind,
|
||||
account: String,
|
||||
username: String,
|
||||
parent_user: String,
|
||||
credential_groups: Option<Vec<String>>,
|
||||
group_memberships: Vec<String>,
|
||||
claims: Option<HashMap<String, Value>>,
|
||||
is_owner: bool,
|
||||
}
|
||||
|
||||
impl RustfsPrincipal {
|
||||
/// The kind of credential the caller authenticated with.
|
||||
#[must_use]
|
||||
pub fn kind(&self) -> PrincipalKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
/// The access key the caller signed with (IAM `Args::account`).
|
||||
#[must_use]
|
||||
pub fn account(&self) -> &str {
|
||||
&self.account
|
||||
}
|
||||
|
||||
/// The user the caller acts as: the parent user for an STS credential or a service account,
|
||||
/// the access key otherwise.
|
||||
#[must_use]
|
||||
pub fn username(&self) -> &str {
|
||||
&self.username
|
||||
}
|
||||
|
||||
/// The parent user IAM records for the credential; empty for root and long-term users.
|
||||
#[must_use]
|
||||
pub fn parent_user(&self) -> &str {
|
||||
&self.parent_user
|
||||
}
|
||||
|
||||
/// The groups recorded on the credential itself (IAM `Args::groups`).
|
||||
#[must_use]
|
||||
pub fn credential_groups(&self) -> &Option<Vec<String>> {
|
||||
&self.credential_groups
|
||||
}
|
||||
|
||||
/// The groups IAM currently lists for [`Self::username`], read once with the snapshot.
|
||||
#[must_use]
|
||||
pub fn group_memberships(&self) -> &[String] {
|
||||
&self.group_memberships
|
||||
}
|
||||
|
||||
/// The claims the credential's token carries; `None` when it has none.
|
||||
#[must_use]
|
||||
pub fn claims(&self) -> Option<&HashMap<String, Value>> {
|
||||
self.claims.as_ref()
|
||||
}
|
||||
|
||||
/// Whether the caller owns the deployment (IAM `Args::is_owner`): root, or a credential of
|
||||
/// root that no session policy or external OIDC identity narrows.
|
||||
#[must_use]
|
||||
pub fn is_owner(&self) -> bool {
|
||||
self.is_owner
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`PolicySource`] over RustFS IAM and the live root credentials.
|
||||
///
|
||||
/// Built only by [`crate::IamAdapters::new`], so it always reads the same [`RootSource`] as the
|
||||
/// credential provider it was assembled with.
|
||||
pub struct IamPolicySource {
|
||||
iam: Arc<dyn IamPrincipalLookup>,
|
||||
root: Arc<dyn RootSource>,
|
||||
}
|
||||
|
||||
impl IamPolicySource {
|
||||
pub(crate) fn new(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) -> Self {
|
||||
Self { iam, root }
|
||||
}
|
||||
|
||||
async fn read(&self, identity: Option<&Identity>) -> Result<PolicySnapshot, PolicyError> {
|
||||
let Some(identity) = identity else {
|
||||
return Ok(PolicySnapshot::empty());
|
||||
};
|
||||
let principal = self.principal(identity).await?;
|
||||
Ok(PolicySnapshot::of(Arc::new(principal)))
|
||||
}
|
||||
|
||||
async fn principal(&self, identity: &Identity) -> Result<RustfsPrincipal, PolicyError> {
|
||||
let access_key = identity.access_key_id();
|
||||
let has_session = identity.session().is_some();
|
||||
// Without root credentials neither ownership nor STS claims can be decided.
|
||||
let Some(root) = self.root.current() else {
|
||||
return Err(PolicyError::unavailable());
|
||||
};
|
||||
let root_access_key = root.access_key();
|
||||
|
||||
if access_key == root_access_key {
|
||||
// Root has no session token, so a session identity for its key is not one IAM issued.
|
||||
if has_session {
|
||||
return Err(PolicyError::because(SESSION_MISMATCH));
|
||||
}
|
||||
return Ok(RustfsPrincipal {
|
||||
kind: PrincipalKind::Root,
|
||||
account: access_key.to_owned(),
|
||||
username: access_key.to_owned(),
|
||||
parent_user: String::new(),
|
||||
credential_groups: None,
|
||||
group_memberships: Vec::new(),
|
||||
claims: None,
|
||||
is_owner: true,
|
||||
});
|
||||
}
|
||||
|
||||
if !self.iam.is_ready() {
|
||||
return Err(PolicyError::unavailable());
|
||||
}
|
||||
let cred = match self.iam.check_key(access_key).await {
|
||||
Ok((Some(identity), true)) => identity.credentials,
|
||||
Ok((Some(_), false)) => return Err(PolicyError::because(DISABLED_PRINCIPAL)),
|
||||
Ok((None, _)) => return Err(PolicyError::because(UNKNOWN_PRINCIPAL)),
|
||||
Err(error) => return Err(classify_iam_error(&error)),
|
||||
};
|
||||
if cred.status == ACCOUNT_OFF || !is_usable_secret(cred.secret_key.as_bytes()) {
|
||||
return Err(PolicyError::because(DISABLED_PRINCIPAL));
|
||||
}
|
||||
|
||||
let kind = if cred.is_service_account() {
|
||||
PrincipalKind::ServiceAccount
|
||||
} else if !cred.session_token.is_empty() {
|
||||
PrincipalKind::Temporary
|
||||
} else {
|
||||
PrincipalKind::User
|
||||
};
|
||||
// The gateway admitted this identity against the credential IAM returned, so the kinds
|
||||
// agree unless IAM's record changed in between; refuse rather than judge a stale reading.
|
||||
if has_session != (kind == PrincipalKind::Temporary) {
|
||||
return Err(PolicyError::because(SESSION_MISMATCH));
|
||||
}
|
||||
|
||||
let claims = read_claims(kind, &cred, &root)?;
|
||||
let username = match kind {
|
||||
PrincipalKind::Temporary | PrincipalKind::ServiceAccount => cred.parent_user.clone(),
|
||||
PrincipalKind::Root | PrincipalKind::User => cred.access_key.clone(),
|
||||
};
|
||||
let membership_subject = if username.is_empty() { access_key } else { username.as_str() };
|
||||
let group_memberships = self.iam.group_memberships(membership_subject);
|
||||
let is_owner = acts_for_root(&cred, claims.as_ref(), root_access_key);
|
||||
|
||||
Ok(RustfsPrincipal {
|
||||
kind,
|
||||
account: access_key.to_owned(),
|
||||
username,
|
||||
parent_user: cred.parent_user,
|
||||
credential_groups: cred.groups,
|
||||
group_memberships,
|
||||
claims,
|
||||
is_owner,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The claims of the credential's own token. The gateway admitted the presented token only when it
|
||||
/// equals the stored one, so reading the stored token reads the presented one.
|
||||
fn read_claims(
|
||||
kind: PrincipalKind,
|
||||
cred: &IamCredential,
|
||||
root: &RootCredentials,
|
||||
) -> Result<Option<HashMap<String, Value>>, PolicyError> {
|
||||
// As in legacy `check_claims_from_token_with_context`: no token, no claims.
|
||||
if cred.session_token.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let claims = match kind {
|
||||
PrincipalKind::ServiceAccount => {
|
||||
get_claims_from_token_with_secret_allow_missing_exp(&cred.session_token, &cred.secret_key)
|
||||
.map_err(|_| PolicyError::malformed())?
|
||||
}
|
||||
PrincipalKind::Temporary => get_claims_from_token_with_secret(&cred.session_token, root.sts_signing_key())
|
||||
.map_err(|_| PolicyError::malformed())?,
|
||||
PrincipalKind::Root | PrincipalKind::User => return Ok(None),
|
||||
};
|
||||
Ok(if claims.is_empty() { None } else { Some(claims) })
|
||||
}
|
||||
|
||||
/// Classifies an IAM failure without keeping any of its text.
|
||||
fn classify_iam_error(error: &IamError) -> PolicyError {
|
||||
match error {
|
||||
IamError::NoSuchUser(_)
|
||||
| IamError::NoSuchServiceAccount(_)
|
||||
| IamError::NoSuchTempAccount(_)
|
||||
| IamError::NoSuchAccount(_) => PolicyError::because(UNKNOWN_PRINCIPAL),
|
||||
_ => PolicyError::unavailable(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Root ownership as the legacy edge decides it (`rustfs/src/auth.rs` `has_root_access`): the
|
||||
/// credential is root's or acts for root, and neither a session policy nor an external OIDC
|
||||
/// identity narrows it.
|
||||
fn acts_for_root(cred: &IamCredential, claims: Option<&HashMap<String, Value>>, root_access_key: &str) -> bool {
|
||||
(cred.access_key == root_access_key || cred.parent_user == root_access_key)
|
||||
&& !claims.is_some_and(|claims| claims.contains_key(SESSION_POLICY_NAME) || is_rustfs_oidc_claims(claims))
|
||||
}
|
||||
|
||||
impl PolicySource for IamPolicySource {
|
||||
fn snapshot<'a>(&'a self, identity: Option<&'a Identity>) -> BoxFuture<'a, Result<PolicySnapshot, PolicyError>> {
|
||||
Box::pin(self.read(identity))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::root::RootCell;
|
||||
use rustfs_credentials::{Credentials as IamCredential, IAM_POLICY_CLAIM_NAME_SA};
|
||||
use rustfs_gateway::SessionBinding;
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::sys::SESSION_POLICY_NAME;
|
||||
use rustfs_iam::utils::generate_jwt;
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
const ROOT_ACCESS_KEY: &str = "rootadmin";
|
||||
const ROOT_SECRET: &str = "root-secret-key-0001";
|
||||
/// Deliberately not the root secret, so every STS test proves which key claims are read with.
|
||||
const SIGNING_KEY: &str = "sts-signing-key-0001";
|
||||
const ACCESS_KEY: &str = "AKIDUSER0001";
|
||||
const SECRET: &str = "user-secret-key-0001";
|
||||
const PARENT: &str = "alice";
|
||||
const EXPIRES_AT: i64 = 4_000_000_000;
|
||||
|
||||
type KeyAnswer = Result<(Option<UserIdentity>, bool), IamError>;
|
||||
type Answer = Box<dyn Fn() -> KeyAnswer + Send + Sync>;
|
||||
|
||||
struct FakeIam {
|
||||
ready: bool,
|
||||
answer: Answer,
|
||||
calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl IamKeyLookup for FakeIam {
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, KeyAnswer> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
let answer = (self.answer)();
|
||||
Box::pin(async move { answer })
|
||||
}
|
||||
}
|
||||
|
||||
impl IamPrincipalLookup for FakeIam {
|
||||
/// Distinct groups per name, so a test can tell which name the source asked about.
|
||||
fn group_memberships(&self, user: &str) -> Vec<String> {
|
||||
match user {
|
||||
PARENT => vec!["parent-group".to_owned()],
|
||||
ACCESS_KEY => vec!["key-group".to_owned()],
|
||||
ROOT_ACCESS_KEY => vec!["root-group".to_owned()],
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn fake(ready: bool, answer: impl Fn() -> KeyAnswer + Send + Sync + 'static) -> Arc<FakeIam> {
|
||||
Arc::new(FakeIam {
|
||||
ready,
|
||||
answer: Box::new(answer),
|
||||
calls: AtomicUsize::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
fn root() -> RootCredentials {
|
||||
RootCredentials::new(ROOT_ACCESS_KEY, ROOT_SECRET, SIGNING_KEY).expect("valid root credentials")
|
||||
}
|
||||
|
||||
fn source(iam: Arc<FakeIam>) -> IamPolicySource {
|
||||
IamPolicySource::new(iam, Arc::new(RootCell::new(Some(root()))))
|
||||
}
|
||||
|
||||
fn user() -> IamCredential {
|
||||
IamCredential {
|
||||
access_key: ACCESS_KEY.to_owned(),
|
||||
secret_key: SECRET.to_owned(),
|
||||
status: "on".to_owned(),
|
||||
groups: Some(vec!["cred-group".to_owned()]),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn sts_token(claims: &[(&str, Value)], secret: &str) -> String {
|
||||
let mut map: HashMap<String, Value> = claims.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect();
|
||||
map.insert("exp".to_owned(), Value::from(EXPIRES_AT));
|
||||
generate_jwt(&map, secret).expect("a token")
|
||||
}
|
||||
|
||||
fn sts(parent: &str, token: String) -> IamCredential {
|
||||
IamCredential {
|
||||
session_token: token,
|
||||
parent_user: parent.to_owned(),
|
||||
expiration: Some(OffsetDateTime::from_unix_timestamp(EXPIRES_AT).expect("valid timestamp")),
|
||||
..user()
|
||||
}
|
||||
}
|
||||
|
||||
fn found(cred: IamCredential) -> KeyAnswer {
|
||||
Ok((Some(UserIdentity::new(cred)), true))
|
||||
}
|
||||
|
||||
fn long_term(access_key: &str) -> Identity {
|
||||
Identity::new(access_key).expect("valid access key")
|
||||
}
|
||||
|
||||
fn session(access_key: &str) -> Identity {
|
||||
let binding = SessionBinding::new(crate::STS_SESSION_ISSUER, EXPIRES_AT).expect("valid binding");
|
||||
long_term(access_key).with_session_binding(binding)
|
||||
}
|
||||
|
||||
async fn principal_of(source: &IamPolicySource, identity: &Identity) -> PolicySnapshot {
|
||||
source.snapshot(Some(identity)).await.expect("a snapshot")
|
||||
}
|
||||
|
||||
fn payload(snapshot: &PolicySnapshot) -> &RustfsPrincipal {
|
||||
snapshot.get::<RustfsPrincipal>().expect("the payload is a RustfsPrincipal")
|
||||
}
|
||||
|
||||
fn refusal(result: Result<PolicySnapshot, PolicyError>) -> PolicyError {
|
||||
match result {
|
||||
Ok(snapshot) => panic!("expected a refusal, got a snapshot (empty: {})", snapshot.is_empty()),
|
||||
Err(error) => error,
|
||||
}
|
||||
}
|
||||
|
||||
/// Compile-time: `RustfsPrincipal` has no `Debug`, so no formatting of a snapshot payload can
|
||||
/// reach its claims. The two blanket impls make the path ambiguous, and so a compile error, as
|
||||
/// soon as `RustfsPrincipal: Debug` holds.
|
||||
#[test]
|
||||
fn policy_principal_has_no_debug() {
|
||||
trait AmbiguousIfDebug<A> {
|
||||
fn some_item() {}
|
||||
}
|
||||
impl<T: ?Sized> AmbiguousIfDebug<()> for T {}
|
||||
impl<T: ?Sized + core::fmt::Debug> AmbiguousIfDebug<u8> for T {}
|
||||
let _ = <RustfsPrincipal as AmbiguousIfDebug<_>>::some_item;
|
||||
}
|
||||
|
||||
/// Positive — a long-term user's facts: itself as username, both group sources, no claims,
|
||||
/// not the owner.
|
||||
#[tokio::test]
|
||||
async fn policy_user_snapshot_carries_iam_facts() {
|
||||
let iam = fake(true, || found(user()));
|
||||
let snapshot = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
|
||||
let principal = payload(&snapshot);
|
||||
assert!(principal.kind() == PrincipalKind::User);
|
||||
assert_eq!(principal.account(), ACCESS_KEY);
|
||||
assert_eq!(principal.username(), ACCESS_KEY);
|
||||
assert_eq!(principal.parent_user(), "");
|
||||
assert_eq!(principal.credential_groups(), &Some(vec!["cred-group".to_owned()]));
|
||||
assert_eq!(principal.group_memberships(), ["key-group".to_owned()]);
|
||||
assert!(principal.claims().is_none());
|
||||
assert!(!principal.is_owner());
|
||||
}
|
||||
|
||||
/// Positive — an STS credential acts for its parent user, and its claims are the ones its
|
||||
/// token carries, read with the root secret that signed it.
|
||||
#[tokio::test]
|
||||
async fn policy_sts_snapshot_carries_parent_user_and_claims() {
|
||||
let token = sts_token(&[("sub", Value::from("alice-session"))], SIGNING_KEY);
|
||||
let iam = fake(true, move || found(sts(PARENT, token.clone())));
|
||||
let snapshot = principal_of(&source(iam), &session(ACCESS_KEY)).await;
|
||||
let principal = payload(&snapshot);
|
||||
assert!(principal.kind() == PrincipalKind::Temporary);
|
||||
assert_eq!(principal.account(), ACCESS_KEY);
|
||||
assert_eq!(principal.parent_user(), PARENT);
|
||||
assert_eq!(principal.username(), PARENT);
|
||||
let claims = principal.claims().expect("an STS principal has claims");
|
||||
assert_eq!(claims.get("sub"), Some(&Value::from("alice-session")));
|
||||
assert_eq!(principal.group_memberships(), ["parent-group".to_owned()]);
|
||||
assert!(!principal.is_owner());
|
||||
}
|
||||
|
||||
/// Positive — a service account's claims are read with its own secret, and it acts for its
|
||||
/// parent.
|
||||
#[tokio::test]
|
||||
async fn policy_service_account_snapshot_reads_its_own_token() {
|
||||
let mut token_claims: HashMap<String, Value> = HashMap::new();
|
||||
token_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"));
|
||||
token_claims.insert("parent".to_owned(), Value::from(PARENT));
|
||||
let token = generate_jwt(&token_claims, SECRET).expect("a token");
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user();
|
||||
cred.session_token = token.clone();
|
||||
cred.parent_user = PARENT.to_owned();
|
||||
cred.claims = Some([(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"))].into());
|
||||
found(cred)
|
||||
});
|
||||
let snapshot = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
|
||||
let principal = payload(&snapshot);
|
||||
assert!(principal.kind() == PrincipalKind::ServiceAccount);
|
||||
assert_eq!(principal.username(), PARENT);
|
||||
assert_eq!(principal.claims().and_then(|c| c.get("parent")), Some(&Value::from(PARENT)));
|
||||
assert_eq!(principal.group_memberships(), ["parent-group".to_owned()]);
|
||||
assert!(!principal.is_owner());
|
||||
}
|
||||
|
||||
/// Positive and negative — root is the owner without asking IAM, even before IAM is ready;
|
||||
/// an STS credential of root is the owner too, but not once a session policy narrows it.
|
||||
#[tokio::test]
|
||||
async fn policy_owner_is_root_or_unrestricted_root_session() {
|
||||
let not_ready = fake(false, || found(user()));
|
||||
let snapshot = principal_of(&source(not_ready.clone()), &long_term(ROOT_ACCESS_KEY)).await;
|
||||
let root = payload(&snapshot);
|
||||
assert!(root.kind() == PrincipalKind::Root);
|
||||
assert!(root.is_owner());
|
||||
assert_eq!(not_ready.calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
let plain = sts_token(&[], SIGNING_KEY);
|
||||
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, plain.clone())));
|
||||
assert!(payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
|
||||
|
||||
let narrowed = sts_token(&[(SESSION_POLICY_NAME, Value::from("e30"))], SIGNING_KEY);
|
||||
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, narrowed.clone())));
|
||||
assert!(!payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
|
||||
}
|
||||
|
||||
/// Negative — an anonymous caller is an empty snapshot, and IAM is not asked.
|
||||
#[tokio::test]
|
||||
async fn policy_anonymous_is_empty_snapshot() {
|
||||
let iam = fake(true, || found(user()));
|
||||
let snapshot = source(iam.clone()).snapshot(None).await.expect("an anonymous snapshot");
|
||||
assert!(snapshot.is_empty());
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// Negative — IAM that is not ready, not initialized, or failing is `unavailable`: never an
|
||||
/// empty snapshot, which an authorizer would judge as an anonymous caller.
|
||||
#[tokio::test]
|
||||
async fn policy_iam_unavailable_is_unavailable() {
|
||||
let not_ready = fake(false, || found(user()));
|
||||
let error = refusal(source(not_ready.clone()).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::unavailable());
|
||||
assert_eq!(not_ready.calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
let failures: [fn() -> IamError; 2] = [
|
||||
|| IamError::IamSysNotInitialized,
|
||||
|| IamError::StringError("storage read failed".to_owned()),
|
||||
];
|
||||
for failure in failures {
|
||||
let iam = fake(true, move || Err(failure()));
|
||||
let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::unavailable(), "{:?}", failure());
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — a principal IAM no longer knows is refused as unknown, not as unavailable and
|
||||
/// not as an empty snapshot.
|
||||
#[tokio::test]
|
||||
async fn policy_unknown_principal_is_refused() {
|
||||
let answers: [fn() -> KeyAnswer; 2] = [|| Ok((None, false)), || Err(IamError::NoSuchUser(ACCESS_KEY.to_owned()))];
|
||||
for answer in answers {
|
||||
let error = refusal(source(fake(true, answer)).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(UNKNOWN_PRINCIPAL));
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — a disabled or invalid principal is refused rather than given a snapshot.
|
||||
#[tokio::test]
|
||||
async fn policy_disabled_principal_is_refused() {
|
||||
let invalid = fake(true, || Ok((Some(UserIdentity::new(user())), false)));
|
||||
let error = refusal(source(invalid).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL));
|
||||
|
||||
let off = fake(true, || {
|
||||
let mut cred = user();
|
||||
cred.status = "off".to_owned();
|
||||
found(cred)
|
||||
});
|
||||
let error = refusal(source(off).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL));
|
||||
}
|
||||
|
||||
/// Negative — an STS token not signed by root yields no claims at all: it is `malformed`.
|
||||
#[tokio::test]
|
||||
async fn policy_sts_token_with_foreign_signature_is_malformed() {
|
||||
let forged = sts_token(&[("sub", Value::from("alice-session"))], "not-the-root-secret");
|
||||
let iam = fake(true, move || found(sts(PARENT, forged.clone())));
|
||||
let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::malformed());
|
||||
}
|
||||
|
||||
/// Negative — the identity's credential kind must match what IAM holds: a session identity for
|
||||
/// a long-term key, or a long-term identity for an STS key, is refused.
|
||||
#[tokio::test]
|
||||
async fn policy_session_kind_mismatch_is_refused() {
|
||||
let token = sts_token(&[], SIGNING_KEY);
|
||||
let iam = fake(true, move || found(sts(PARENT, token.clone())));
|
||||
let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
|
||||
|
||||
let iam = fake(true, || found(user()));
|
||||
let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
|
||||
|
||||
let iam = fake(false, || found(user()));
|
||||
let error = refusal(source(iam).snapshot(Some(&session(ROOT_ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
|
||||
}
|
||||
|
||||
fn service_account(parent: &str, extra_claims: &[(&str, Value)]) -> KeyAnswer {
|
||||
let mut token_claims: HashMap<String, Value> = extra_claims.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect();
|
||||
token_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"));
|
||||
let mut cred = user();
|
||||
cred.session_token = generate_jwt(&token_claims, SECRET).expect("a token");
|
||||
cred.parent_user = parent.to_owned();
|
||||
cred.claims = Some([(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"))].into());
|
||||
found(cred)
|
||||
}
|
||||
|
||||
/// Negative — a valid-flagged identity whose secret is empty or short gets no snapshot.
|
||||
#[tokio::test]
|
||||
async fn policy_short_secret_principal_is_refused() {
|
||||
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user();
|
||||
cred.secret_key = weak.to_owned();
|
||||
found(cred)
|
||||
});
|
||||
let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL), "{weak:?}");
|
||||
}
|
||||
for usable in ["12345678", "abc\0defg"] {
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user();
|
||||
cred.secret_key = usable.to_owned();
|
||||
found(cred)
|
||||
});
|
||||
assert!(payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).kind() == PrincipalKind::User);
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — a root session issued through external OIDC is not the owner, though its parent
|
||||
/// is root; the same session without the OIDC issuer is.
|
||||
#[tokio::test]
|
||||
async fn policy_oidc_root_session_is_not_owner() {
|
||||
let oidc = sts_token(
|
||||
&[
|
||||
("iss", Value::from("rustfs-oidc")),
|
||||
("oidc_provider", Value::from("corp-idp")),
|
||||
("sub", Value::from("alice@corp")),
|
||||
],
|
||||
SIGNING_KEY,
|
||||
);
|
||||
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, oidc.clone())));
|
||||
assert!(!payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
|
||||
|
||||
let local = sts_token(
|
||||
&[
|
||||
("iss", Value::from("rustfs-sts")),
|
||||
("oidc_provider", Value::from("corp-idp")),
|
||||
("sub", Value::from("alice@corp")),
|
||||
],
|
||||
SIGNING_KEY,
|
||||
);
|
||||
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, local.clone())));
|
||||
assert!(payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
|
||||
}
|
||||
|
||||
/// Positive and negative — a service account of root is the owner; one narrowed by a session
|
||||
/// policy is not, and neither is a service account of an ordinary user.
|
||||
#[tokio::test]
|
||||
async fn policy_service_account_of_root_is_owner() {
|
||||
let iam = fake(true, || service_account(ROOT_ACCESS_KEY, &[]));
|
||||
let principal = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
|
||||
assert!(payload(&principal).kind() == PrincipalKind::ServiceAccount);
|
||||
assert!(payload(&principal).is_owner());
|
||||
|
||||
let iam = fake(true, || service_account(ROOT_ACCESS_KEY, &[(SESSION_POLICY_NAME, Value::from("e30"))]));
|
||||
assert!(!payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).is_owner());
|
||||
|
||||
let iam = fake(true, || service_account(PARENT, &[]));
|
||||
assert!(!payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).is_owner());
|
||||
}
|
||||
|
||||
/// Negative — STS claims are read with the STS signing key, not with the root secret: a token
|
||||
/// signed with the root secret while the signing key differs is `malformed`.
|
||||
#[tokio::test]
|
||||
async fn policy_sts_claims_use_the_signing_key() {
|
||||
let root_signed = sts_token(&[("sub", Value::from("alice-session"))], ROOT_SECRET);
|
||||
let iam = fake(true, move || found(sts(PARENT, root_signed.clone())));
|
||||
let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::malformed());
|
||||
}
|
||||
|
||||
/// Negative — without root credentials no principal can be judged: `unavailable`, never an
|
||||
/// empty snapshot, and IAM is not asked.
|
||||
#[tokio::test]
|
||||
async fn policy_without_root_is_unavailable() {
|
||||
let iam = fake(true, || found(user()));
|
||||
let source = IamPolicySource::new(iam.clone(), Arc::new(RootCell::new(None)));
|
||||
for identity in [long_term(ACCESS_KEY), long_term(ROOT_ACCESS_KEY)] {
|
||||
assert_eq!(refusal(source.snapshot(Some(&identity)).await), PolicyError::unavailable());
|
||||
}
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
type LiveRoot = Arc<std::sync::Mutex<Option<(String, String)>>>;
|
||||
|
||||
/// The primary path: a source that reads a live accessor on every call, as the binary's does,
|
||||
/// with nothing cached in between.
|
||||
fn live_root_source(live: &LiveRoot) -> Arc<dyn RootSource> {
|
||||
let accessor = live.clone();
|
||||
Arc::new(move || {
|
||||
let guard = accessor.lock().ok()?;
|
||||
let (access_key, secret_key) = guard.as_ref()?;
|
||||
RootCredentials::new(access_key, secret_key, SIGNING_KEY).ok().map(Arc::new)
|
||||
})
|
||||
}
|
||||
|
||||
fn set_live(live: &LiveRoot, value: Option<(&str, &str)>) {
|
||||
*live.lock().expect("live root lock") = value.map(|(ak, sk)| (ak.to_owned(), sk.to_owned()));
|
||||
}
|
||||
|
||||
/// Positive and negative — the two adapters read one live root accessor: a root change is seen
|
||||
/// by both on the next request with no second step, the old root key stops being root to
|
||||
/// both, and an uninitialized or unusable root makes both unavailable.
|
||||
#[tokio::test]
|
||||
async fn adapters_share_one_live_root() {
|
||||
use rustfs_gateway::{CredentialLookup, CredentialProvider, ProviderError};
|
||||
|
||||
let live: LiveRoot = Arc::default();
|
||||
set_live(&live, Some((ROOT_ACCESS_KEY, ROOT_SECRET)));
|
||||
let iam = fake(true, || Ok((None, false)));
|
||||
let adapters = crate::IamAdapters::new(iam, live_root_source(&live));
|
||||
|
||||
let found = adapters.credentials.lookup(ROOT_ACCESS_KEY).await;
|
||||
assert!(matches!(found, Ok(CredentialLookup::Found(_))));
|
||||
let snapshot = principal_of(&adapters.policy, &long_term(ROOT_ACCESS_KEY)).await;
|
||||
assert!(payload(&snapshot).is_owner());
|
||||
|
||||
set_live(&live, Some(("newroot", "new-root-secret-01")));
|
||||
match adapters.credentials.lookup("newroot").await {
|
||||
Ok(CredentialLookup::Found(credentials)) => assert_eq!(credentials.secret().expose(), b"new-root-secret-01"),
|
||||
_ => panic!("the new root key must resolve to the new secret"),
|
||||
}
|
||||
assert!(matches!(
|
||||
adapters.credentials.lookup(ROOT_ACCESS_KEY).await,
|
||||
Ok(CredentialLookup::NotFound)
|
||||
));
|
||||
assert!(payload(&principal_of(&adapters.policy, &long_term("newroot")).await).is_owner());
|
||||
let old = refusal(adapters.policy.snapshot(Some(&long_term(ROOT_ACCESS_KEY))).await);
|
||||
assert_eq!(old, PolicyError::because(UNKNOWN_PRINCIPAL));
|
||||
|
||||
for unusable in [None, Some(("newroot", "\0\0\0\0\0\0\0\0"))] {
|
||||
set_live(&live, unusable);
|
||||
assert!(matches!(adapters.credentials.lookup("newroot").await, Err(ProviderError::Unavailable)));
|
||||
assert_eq!(
|
||||
refusal(adapters.policy.snapshot(Some(&long_term("newroot"))).await),
|
||||
PolicyError::unavailable()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive — `install` builds the authenticator exactly once, over a provider that reads the
|
||||
/// pair's root source.
|
||||
#[tokio::test]
|
||||
async fn adapters_install_hands_over_the_paired_provider_once() {
|
||||
use rustfs_gateway::{CredentialLookup, CredentialProvider, RegionSet, ServiceBuilder, SigV4Authenticator};
|
||||
|
||||
let live: LiveRoot = Arc::default();
|
||||
set_live(&live, Some((ROOT_ACCESS_KEY, ROOT_SECRET)));
|
||||
let adapters = crate::IamAdapters::new(fake(true, || Ok((None, false))), live_root_source(&live));
|
||||
let mut handed = Vec::new();
|
||||
let _builder = adapters.install(ServiceBuilder::new(), |provider| {
|
||||
handed.push(provider.clone());
|
||||
SigV4Authenticator::new(provider, RegionSet::new(["us-east-1"]).expect("a region"))
|
||||
});
|
||||
assert_eq!(handed.len(), 1);
|
||||
let found = handed[0].lookup(ROOT_ACCESS_KEY).await;
|
||||
assert!(matches!(found, Ok(CredentialLookup::Found(_))));
|
||||
}
|
||||
}
|
||||
@@ -1,216 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The deployment's root credentials, as every adapter in this crate reads them.
|
||||
//!
|
||||
//! Responsible for: [`RootCredentials`], the validated root key pair plus the key STS session
|
||||
//! tokens are signed with; [`RootSource`], the one live handle both the credential provider and
|
||||
//! the policy source read on every request; and the effective-secret-length rule every secret in
|
||||
//! this crate is held to.
|
||||
//! NOT responsible for: where the root keys come from or when they change. The binary implements
|
||||
//! [`RootSource`] directly over its live root accessor — a closure is enough — so there is no
|
||||
//! second copy of root to keep in step.
|
||||
//! Upstream: the binary's runtime credentials. Downstream: `crate::credentials`, `crate::policy`.
|
||||
//!
|
||||
//! # Why one source
|
||||
//!
|
||||
//! Legacy reads the current root credentials on every request. Two copies taken at assembly could
|
||||
//! disagree, or go stale after a root change, and a key one adapter still treats as root would be
|
||||
//! answered with the root secret or judged as the owner. Both adapters therefore hold the same
|
||||
//! `Arc<dyn RootSource>`, handed to them together by [`crate::IamAdapters::new`].
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::Credentials;
|
||||
|
||||
/// The shortest effective secret key, in bytes, that any credential in this crate may verify a
|
||||
/// signature with. The same bound IAM applies to its own credentials; a signature over a shorter
|
||||
/// (or empty) key is one anybody can compute.
|
||||
pub const MIN_SECRET_KEY_LEN: usize = 8;
|
||||
|
||||
/// The number of bytes of `secret` a signature actually depends on: its length after trailing NUL
|
||||
/// bytes are removed.
|
||||
///
|
||||
/// HMAC pads a key shorter than its block with zero bytes, and SigV4's first key is `"AWS4"`
|
||||
/// followed by the secret, so trailing NULs add nothing: eight NUL bytes sign exactly like the
|
||||
/// empty secret, and `"abc"` followed by five NULs signs like `"abc"`. A secret of NULs only has
|
||||
/// an effective length of zero. An interior NUL is followed by real key bytes, which change the
|
||||
/// padded key, so it counts like any other byte.
|
||||
pub(crate) fn effective_secret_len(secret: &[u8]) -> usize {
|
||||
secret.iter().rposition(|byte| *byte != 0).map_or(0, |last| last + 1)
|
||||
}
|
||||
|
||||
/// Whether `secret` may verify a signature: its [`effective_secret_len`] is at least
|
||||
/// [`MIN_SECRET_KEY_LEN`]. The one rule for root, IAM credentials and the STS signing key.
|
||||
pub(crate) fn is_usable_secret(secret: &[u8]) -> bool {
|
||||
effective_secret_len(secret) >= MIN_SECRET_KEY_LEN
|
||||
}
|
||||
|
||||
/// Why [`RootCredentials::new`] refused its input. Carries no input text.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RootCredentialsError {
|
||||
/// The access key id is not one the gateway can hold.
|
||||
InvalidAccessKey,
|
||||
/// The root secret key fails the effective-length rule.
|
||||
WeakSecretKey,
|
||||
/// The STS token-signing key fails the effective-length rule.
|
||||
WeakSigningKey,
|
||||
}
|
||||
|
||||
impl core::fmt::Display for RootCredentialsError {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::InvalidAccessKey => "root access key is not usable",
|
||||
Self::WeakSecretKey => "root secret key is shorter than the minimum length",
|
||||
Self::WeakSigningKey => "STS token-signing key is shorter than the minimum length",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for RootCredentialsError {}
|
||||
|
||||
/// The root key pair and the STS token-signing key, validated together.
|
||||
///
|
||||
/// Implements no `Debug`; the secrets are readable only inside this crate.
|
||||
pub struct RootCredentials {
|
||||
credentials: Credentials,
|
||||
sts_signing_key: Box<str>,
|
||||
}
|
||||
|
||||
impl RootCredentials {
|
||||
/// Validates the root credentials and the key IAM signs STS session tokens with (today the
|
||||
/// root secret, `rustfs_iam::manager::get_token_signing_key`; pass it separately so a future
|
||||
/// key separation changes no adapter).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`RootCredentialsError`] when either secret fails the effective-length rule (fewer than
|
||||
/// [`MIN_SECRET_KEY_LEN`] bytes once trailing NUL bytes are removed) or the access key id is not
|
||||
/// one the gateway can hold.
|
||||
pub fn new(access_key: &str, secret_key: &str, sts_signing_key: &str) -> Result<Self, RootCredentialsError> {
|
||||
if !is_usable_secret(secret_key.as_bytes()) {
|
||||
return Err(RootCredentialsError::WeakSecretKey);
|
||||
}
|
||||
if !is_usable_secret(sts_signing_key.as_bytes()) {
|
||||
return Err(RootCredentialsError::WeakSigningKey);
|
||||
}
|
||||
let credentials =
|
||||
Credentials::new(access_key, secret_key.as_bytes()).map_err(|_| RootCredentialsError::InvalidAccessKey)?;
|
||||
Ok(Self {
|
||||
credentials,
|
||||
sts_signing_key: sts_signing_key.into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The root access key id, a public identifier.
|
||||
#[must_use]
|
||||
pub fn access_key(&self) -> &str {
|
||||
self.credentials.identity().access_key_id()
|
||||
}
|
||||
|
||||
pub(crate) fn credentials(&self) -> &Credentials {
|
||||
&self.credentials
|
||||
}
|
||||
|
||||
pub(crate) fn sts_signing_key(&self) -> &str {
|
||||
&self.sts_signing_key
|
||||
}
|
||||
}
|
||||
|
||||
/// The live root credentials, read on every request by both adapters.
|
||||
///
|
||||
/// The binary implements it directly over RustFS's live root accessor, so a root change needs no
|
||||
/// second step. Any `Fn() -> Option<Arc<RootCredentials>>` is a source; for example, over the
|
||||
/// server context:
|
||||
///
|
||||
/// ```ignore
|
||||
/// let root = move || {
|
||||
/// let cred = ctx.action_credentials().get()?;
|
||||
/// let signing_key = ctx.token_signing_key()?;
|
||||
/// RootCredentials::new(&cred.access_key, &cred.secret_key, &signing_key).ok().map(Arc::new)
|
||||
/// };
|
||||
/// let adapters = IamAdapters::new(iam, Arc::new(root));
|
||||
/// ```
|
||||
///
|
||||
/// Root credentials that fail validation read as `None`, which both adapters answer as
|
||||
/// unavailable.
|
||||
pub trait RootSource: Send + Sync + 'static {
|
||||
/// The current root credentials; `None` while they are not initialized, which both adapters
|
||||
/// answer as unavailable rather than guessing.
|
||||
fn current(&self) -> Option<Arc<RootCredentials>>;
|
||||
}
|
||||
|
||||
impl<F> RootSource for F
|
||||
where
|
||||
F: Fn() -> Option<Arc<RootCredentials>> + Send + Sync + 'static,
|
||||
{
|
||||
fn current(&self) -> Option<Arc<RootCredentials>> {
|
||||
self()
|
||||
}
|
||||
}
|
||||
|
||||
/// Test helper: a [`RootSource`] holding fixed root credentials. Production code reads the live
|
||||
/// accessor instead, so that there is no copy of root to keep in step.
|
||||
#[cfg(test)]
|
||||
pub(crate) struct RootCell(Option<Arc<RootCredentials>>);
|
||||
|
||||
#[cfg(test)]
|
||||
impl RootCell {
|
||||
pub(crate) fn new(root: Option<RootCredentials>) -> Self {
|
||||
Self(root.map(Arc::new))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl RootSource for RootCell {
|
||||
fn current(&self) -> Option<Arc<RootCredentials>> {
|
||||
self.0.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Negative — an empty or short root secret is refused at construction, so no source can ever
|
||||
/// hand out a root key anybody could sign with; eight bytes is accepted.
|
||||
#[test]
|
||||
fn root_short_secret_is_refused() {
|
||||
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0", "abcdefg\0"] {
|
||||
let refused = RootCredentials::new("rootadmin", weak, "signing-key-0001");
|
||||
assert!(matches!(refused, Err(RootCredentialsError::WeakSecretKey)), "{} bytes", weak.len());
|
||||
}
|
||||
assert!(RootCredentials::new("rootadmin", "12345678", "signing-key-0001").is_ok());
|
||||
}
|
||||
|
||||
/// Positive — an interior NUL is followed by real key bytes and counts: `"abc\0defg"` has an
|
||||
/// effective length of eight and is accepted, as is the same secret with trailing NULs.
|
||||
#[test]
|
||||
fn root_interior_nul_counts_toward_length() {
|
||||
for secret in ["abc\0defg", "abc\0defg\0\0"] {
|
||||
assert!(RootCredentials::new("rootadmin", secret, "signing-key-0001").is_ok(), "{secret:?}");
|
||||
assert!(RootCredentials::new("rootadmin", "root-secret-key-0001", secret).is_ok(), "{secret:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — an empty or short STS signing key is refused at construction.
|
||||
#[test]
|
||||
fn root_short_signing_key_is_refused() {
|
||||
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
|
||||
let refused = RootCredentials::new("rootadmin", "root-secret-key-0001", weak);
|
||||
assert!(matches!(refused, Err(RootCredentialsError::WeakSigningKey)), "{} bytes", weak.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -425,10 +425,7 @@ enum FolderScanSource {
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum FolderScanMode {
|
||||
Namespace,
|
||||
ErasureCandidate {
|
||||
reuse_namespace: bool,
|
||||
parent_has_metadata: bool,
|
||||
},
|
||||
ErasureCandidate { reuse_namespace: bool },
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
@@ -2068,9 +2065,7 @@ impl FolderScanner {
|
||||
if candidate_unclassified {
|
||||
// Apply namespace reuse only after the current listing
|
||||
// proves there is no direct erasure part.
|
||||
if let FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: true, ..
|
||||
} = mode
|
||||
if let FolderScanMode::ErasureCandidate { reuse_namespace: true } = mode
|
||||
&& !into.compacted
|
||||
&& self.old_cache.find(&folder.name).is_some()
|
||||
{
|
||||
@@ -2084,23 +2079,10 @@ impl FolderScanner {
|
||||
return Ok(FolderScanOutcome::Namespace);
|
||||
}
|
||||
candidate_unclassified = false;
|
||||
// An empty UUID directory beside object metadata has
|
||||
// no namespace work. Its listing was already charged;
|
||||
// nonempty namespaces still need directory admission.
|
||||
let empty_object_child = matches!(
|
||||
mode,
|
||||
FolderScanMode::ErasureCandidate {
|
||||
parent_has_metadata: true,
|
||||
..
|
||||
}
|
||||
) && deferred_metadata.is_none()
|
||||
&& queued_folders.is_empty();
|
||||
if !empty_object_child {
|
||||
if !self.budget.try_start_directory() {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
}
|
||||
wait_for_scanner_io(&ctx, self.sleeper.sleep_folder_preset()).await?;
|
||||
if !self.budget.try_start_directory() {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
}
|
||||
wait_for_scanner_io(&ctx, self.sleeper.sleep_folder_preset()).await?;
|
||||
}
|
||||
let Some(entry) = deferred_metadata.take() else { break };
|
||||
(entry, true)
|
||||
@@ -2570,6 +2552,20 @@ impl FolderScanner {
|
||||
return Err(ScannerError::Other("Operation cancelled".to_string()));
|
||||
}
|
||||
|
||||
if found_object_metadata && self.is_erasure_mode {
|
||||
// If we found an object in erasure mode, we skip subdirs (only datadirs)...
|
||||
debug!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_FOLDER_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
folder = %folder.name,
|
||||
state = "erasure_object_found",
|
||||
"Scanner folder descent stopped after erasure object"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
// Checkpointed sweeps need child entries to validate their resume
|
||||
// frontier (#8292). Outside that, also compact a wide bucket root:
|
||||
// excluding it left #8961's ~170k top-level prefixes unbounded until
|
||||
@@ -2658,7 +2654,6 @@ impl FolderScanner {
|
||||
let mut child_mode = if self.is_erasure_mode && is_erasure_candidate(&folder_item.name) {
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: reuse_covered,
|
||||
parent_has_metadata: found_object_metadata,
|
||||
}
|
||||
} else {
|
||||
FolderScanMode::Namespace
|
||||
@@ -2726,10 +2721,7 @@ impl FolderScanner {
|
||||
}
|
||||
// Sampling may skip namespace work, never the
|
||||
// classification that protects its parent object.
|
||||
child_mode = FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: true,
|
||||
parent_has_metadata: found_object_metadata,
|
||||
};
|
||||
child_mode = FolderScanMode::ErasureCandidate { reuse_namespace: true };
|
||||
} else {
|
||||
folder_item.object_heal_prob_div = data_usage_update_dir_cycles();
|
||||
}
|
||||
@@ -2824,11 +2816,6 @@ impl FolderScanner {
|
||||
updates.cache.delete_recursive(&h);
|
||||
}
|
||||
into.children.remove(&clone_cache_value(&h.0));
|
||||
// Object keys can also be prefixes. A classified payload
|
||||
// directory must not hide their other namespace children.
|
||||
if found_object_metadata {
|
||||
continue;
|
||||
}
|
||||
// Earlier namespace siblings may have completed, but none
|
||||
// can certify coverage of this unresolved parent object.
|
||||
self.coverage_gap |= forward_sweep;
|
||||
|
||||
@@ -183,117 +183,6 @@ fn charged_ops(root: &std::path::Path) -> u64 {
|
||||
status.charged_ops
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn erasure_object_metadata_preserves_child_object_namespaces() {
|
||||
for (corrupt_parent, cached_parent) in [(false, false), (true, false), (false, true)] {
|
||||
let (mut scanner, root) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, root.clone());
|
||||
scanner.old_cache.info.name = "bucket".into();
|
||||
scanner.new_cache.info.name = "bucket".into();
|
||||
scanner.is_erasure_mode = true;
|
||||
bind_budget(&mut scanner, &root, 1_000_000);
|
||||
|
||||
let namespace = format!("parent/{}/nested", Uuid::new_v4());
|
||||
for object in ["parent", "parent/child", namespace.as_str()] {
|
||||
write_test_object_metadata_bytes(&root, "bucket", object, &metadata_for_object_version("bucket", object, None)).await;
|
||||
}
|
||||
let payload = root.join("bucket/parent").join(Uuid::new_v4().to_string());
|
||||
tokio::fs::create_dir_all(&payload)
|
||||
.await
|
||||
.expect("create parent payload directory");
|
||||
tokio::fs::write(payload.join("part.1"), b"shard")
|
||||
.await
|
||||
.expect("create actual payload evidence");
|
||||
if corrupt_parent {
|
||||
tokio::fs::write(root.join("bucket/parent/xl.meta"), b"")
|
||||
.await
|
||||
.expect("corrupt only the parent metadata");
|
||||
}
|
||||
if cached_parent {
|
||||
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
|
||||
scanner.old_cache.replace(
|
||||
"bucket/parent",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 1,
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let interval = data_usage_update_dir_cycles();
|
||||
scanner.old_cache.info.next_cycle = u64::from(
|
||||
(0..interval)
|
||||
.find(|cycle| hash_path("bucket/parent").mod_(*cycle, interval))
|
||||
.expect("one normal sampling cycle revisits the old parent-only cache"),
|
||||
);
|
||||
}
|
||||
|
||||
let mut into = DataUsageEntry::default();
|
||||
scanner
|
||||
.scan_folder(
|
||||
CancellationToken::new(),
|
||||
CachedFolder {
|
||||
name: "bucket".into(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut into,
|
||||
)
|
||||
.await
|
||||
.expect("scan object and child namespaces independently");
|
||||
let total = scanner.new_cache.size_recursive("bucket").expect("retain bucket accounting");
|
||||
assert_eq!(total.objects, if corrupt_parent { 2 } else { 3 });
|
||||
assert_eq!(total.size, if corrupt_parent { 2 } else { 3 });
|
||||
assert_eq!(total.failed_objects, usize::from(corrupt_parent));
|
||||
assert_eq!(
|
||||
scanner.new_cache.info.failed_objects.len(),
|
||||
usize::from(corrupt_parent),
|
||||
"payload evidence must not invent missing parent metadata"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn erasure_object_prefix_children_keep_the_object_budget() {
|
||||
let (mut scanner, root) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, root.clone());
|
||||
scanner.old_cache.info.name = "bucket".into();
|
||||
scanner.new_cache.info.name = "bucket".into();
|
||||
scanner.is_erasure_mode = true;
|
||||
bind_budget(&mut scanner, &root, 1_000_000);
|
||||
for object in ["parent", "parent/child", "parent/other"] {
|
||||
write_test_object_metadata(&root, "bucket", object).await;
|
||||
}
|
||||
let visits = metadata_visits::observe(&mut scanner);
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
crate::scanner_budget::ScannerCycleBudgetConfig {
|
||||
max_objects: Some(2),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
scanner.budget = budget.clone();
|
||||
let result = scanner
|
||||
.scan_folder(
|
||||
budget.token(),
|
||||
CachedFolder {
|
||||
name: "bucket".into(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut DataUsageEntry::default(),
|
||||
)
|
||||
.await;
|
||||
assert!(result.is_err(), "parent metadata cannot hide children or bypass their budget");
|
||||
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects));
|
||||
assert_eq!(budget.progress().0, 2);
|
||||
assert_eq!(visits.load(Ordering::Relaxed), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn erasure_directory_probe_reaches_eof_across_budgeted_batches() {
|
||||
@@ -315,10 +204,7 @@ async fn erasure_directory_probe_reaches_eof_across_budgeted_batches() {
|
||||
ctx,
|
||||
folder,
|
||||
&mut DataUsageEntry::default(),
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: false,
|
||||
parent_has_metadata: false,
|
||||
},
|
||||
FolderScanMode::ErasureCandidate { reuse_namespace: false },
|
||||
false,
|
||||
)
|
||||
.await
|
||||
@@ -385,10 +271,7 @@ async fn scanner_directory_probe_budget_wait_cancels_without_rate_delay() {
|
||||
ctx.clone(),
|
||||
folder,
|
||||
&mut DataUsageEntry::default(),
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: false,
|
||||
parent_has_metadata: false,
|
||||
},
|
||||
FolderScanMode::ErasureCandidate { reuse_namespace: false },
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
@@ -428,10 +311,7 @@ async fn scanner_directory_probe_cancels_during_entry_budget_wait() {
|
||||
ctx.clone(),
|
||||
folder,
|
||||
&mut DataUsageEntry::default(),
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: false,
|
||||
parent_has_metadata: false,
|
||||
},
|
||||
FolderScanMode::ErasureCandidate { reuse_namespace: false },
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
@@ -502,16 +382,7 @@ async fn erasure_candidate_defers_metadata_until_part_classification_finishes()
|
||||
scanner.budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, Default::default());
|
||||
let mut into = DataUsageEntry::default();
|
||||
let outcome = scanner
|
||||
.scan_folder_inner(
|
||||
ctx,
|
||||
folder,
|
||||
&mut into,
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: false,
|
||||
parent_has_metadata: false,
|
||||
},
|
||||
false,
|
||||
)
|
||||
.scan_folder_inner(ctx, folder, &mut into, FolderScanMode::ErasureCandidate { reuse_namespace: false }, false)
|
||||
.await
|
||||
.expect("classify candidate");
|
||||
assert_eq!(outcome, FolderScanOutcome::ErasureData);
|
||||
@@ -723,10 +594,7 @@ async fn erasure_candidate_part_symlink_is_not_data_evidence() {
|
||||
CancellationToken::new(),
|
||||
folder,
|
||||
&mut DataUsageEntry::default(),
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace: false,
|
||||
parent_has_metadata: false,
|
||||
},
|
||||
FolderScanMode::ErasureCandidate { reuse_namespace: false },
|
||||
false,
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -54,7 +54,6 @@ The four site-replication state-writer concurrency proofs reserve the available
|
||||
| PR, non-doc change | `Typos` | `ci.yml` `typos` | Via aggregate | `typos` |
|
||||
| Full selection | `Workspace tests` (serial ILM step) | `ci.yml` `workspace-tests` | Via aggregate | exact `Run ignored ILM integration tests serially` command in the job |
|
||||
| Nightly schedule, manual dispatch | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)`, `Test and Lint (http3)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo nextest run` with the job's `--features` |
|
||||
| Full selection, every event | `Test and Lint (gateway-stack)` | `ci.yml` `test-and-lint-gateway-stack` | Via aggregate | `cargo clippy -p rustfs --all-targets --features gateway-stack -- -D warnings`; `cargo nextest run -p rustfs --features gateway-stack` with the job's filter |
|
||||
| Nightly schedule, manual dispatch, full selection | `Connect Feature Boundaries` | `ci.yml` `connect-feature-boundaries` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail; `scripts/check_offline_enrollment_e2e.sh` |
|
||||
| PR, non-doc change | `Build RustFS Debug Binary` | `ci.yml` `build-rustfs-debug-binary` | Via aggregate; prerequisite for black-box jobs | `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks` (binary plus its `rustfs.e2e.json` sidecar) |
|
||||
| Nightly schedule, manual dispatch, full selection | `io_uring Integration (real)` | `ci.yml` `uring-integration` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
|
||||
|
||||
+1
-16
@@ -56,12 +56,6 @@ required-features = ["swift"]
|
||||
name = "swift_object_integration_test"
|
||||
required-features = ["swift"]
|
||||
|
||||
# Starts an embedded server on the gateway stack, which only a build with the
|
||||
# feature can select.
|
||||
[[test]]
|
||||
name = "embedded_gateway_stack_test"
|
||||
required-features = ["gateway-stack"]
|
||||
|
||||
[features]
|
||||
default = ["ftps", "webdav", "gcs"]
|
||||
fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips", "rustfs-ecstore/fips", "rustfs-protocols/fips", "rustfs-s3-client/fips", "rustfs-targets/fips", "rustfs-utils/fips"]
|
||||
@@ -75,19 +69,12 @@ license = []
|
||||
io-scheduler-debug = [] # Enable debug information in I/O scheduler
|
||||
tracing-chunk-debug = [] # Enable per-chunk tracing in data plane (high noise, for debugging only)
|
||||
full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope", "gcs"]
|
||||
# The debug-only E2E binary also carries the gateway S3 stack, so the gateway smoke
|
||||
# lane runs the same artifact as the legacy one with RUSTFS_S3_STACK as its only change.
|
||||
e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hooks", "gateway-stack"]
|
||||
e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hooks"]
|
||||
# Shortens Connect credentials only in debug E2E builds.
|
||||
connect-e2e-short-credentials = []
|
||||
# Builds the dedicated rustfs-cli-e2e target with a build-time public enrollment root.
|
||||
offline-enrollment-e2e-root = []
|
||||
rio-v2 = ["rustfs-ecstore/rio-v2"]
|
||||
# The rustfs-gateway S3 stack, selected at startup with RUSTFS_S3_STACK=gateway
|
||||
# (rustfs/backlog#2734). Under construction: off by default and never enabled in a
|
||||
# release build; the stack refuses to compile without debug assertions, and
|
||||
# `scripts/check_gateway_stack_release_boundary.sh` keeps it out of release inputs.
|
||||
gateway-stack = ["dep:rustfs-gateway", "dep:rustfs-gateway-adapter"]
|
||||
pyroscope = ["rustfs-obs/pyroscope", "dep:pyroscope", "dep:object", "dep:symbolic-demangle"]
|
||||
# Tokio runtime telemetry. Requires `--cfg tokio_unstable`; use `make build-profiling`.
|
||||
dial9 = ["rustfs-obs/dial9"]
|
||||
@@ -268,8 +255,6 @@ rustfs-notify = { workspace = true }
|
||||
rustfs-obs = { workspace = true }
|
||||
rustfs-policy = { workspace = true }
|
||||
rustfs-protocols = { workspace = true }
|
||||
rustfs-gateway = { workspace = true, optional = true }
|
||||
rustfs-gateway-adapter = { workspace = true, optional = true }
|
||||
rustfs-gateway-types = { workspace = true, features = ["compat-s3s-rustfs"] }
|
||||
rustfs-protos = { workspace = true }
|
||||
rustfs-rio = { workspace = true }
|
||||
|
||||
@@ -165,8 +165,7 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
let signing_secret = req
|
||||
.credentials()
|
||||
.ok_or_else(|| s3::error(S3ErrorCode::AccessDenied, "authentication required"))?
|
||||
.require_secret_key()
|
||||
.map_err(|err| s3::error(S3ErrorCode::InternalError, err.to_string()))?
|
||||
.secret_key()
|
||||
.to_string();
|
||||
let caller = CallerIdentity::resolve_envelope(req.envelope()).await?;
|
||||
let audit = AccountAuditContext::from_envelope(req.envelope());
|
||||
|
||||
@@ -63,10 +63,8 @@ impl Operation for RecoverOrphanedBucketHandler {
|
||||
authorize_admin_envelope(req.envelope(), vec![Action::AdminAction(AdminAction::RecoverOrphanedBucketAction)]).await?;
|
||||
let secret_key = req
|
||||
.credentials()
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "authentication required"))?
|
||||
.require_secret_key()
|
||||
.map(|secret| Zeroizing::new(secret.to_owned()))
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, err.to_string()))?;
|
||||
.map(|credentials| Zeroizing::new(credentials.secret_key().to_owned()))
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "authentication required"))?;
|
||||
let bucket = params
|
||||
.get("bucket")
|
||||
.filter(|bucket| !bucket.is_empty())
|
||||
|
||||
@@ -15,12 +15,11 @@
|
||||
use crate::admin::operation::{AdminRequest, AdminResponse, Operation};
|
||||
use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, S3Router};
|
||||
use crate::admin::runtime_sources::{ServerContextSlot, current_action_credentials, current_object_store_handle, current_region};
|
||||
use crate::admin::storage_api::access::{ReqInfo, authorize_owned_request, authorize_request};
|
||||
use crate::admin::storage_api::access::{ReqInfo, authorize_request};
|
||||
use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::admin::storage_api::contract::list::ListOperations as _;
|
||||
use crate::admin::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use crate::admin::storage_api::object::StorageObjectOptions as ObjectOptions;
|
||||
use crate::app::{RequestEnvelope, RequestHead};
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::error::ApiError;
|
||||
use crate::license::license_check;
|
||||
@@ -496,32 +495,25 @@ async fn authorize_object_zip_download_s3_action(
|
||||
req_info.version_id = None;
|
||||
}
|
||||
|
||||
let mut extensions = http::Extensions::new();
|
||||
extensions.insert(req.remote_addr().map(RemoteAddr));
|
||||
if let Some(client_info) = req.client_info() {
|
||||
extensions.insert(client_info.clone());
|
||||
}
|
||||
if let Some(server_context) = req.envelope().server_context() {
|
||||
extensions.insert(server_context.clone());
|
||||
}
|
||||
if let Some(req_info) = req.req_info() {
|
||||
extensions.insert(req_info.clone());
|
||||
}
|
||||
let mut authorization = RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method: req.method().clone(),
|
||||
uri: req.uri().clone(),
|
||||
headers: req.headers().clone(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
extensions,
|
||||
let mut authorization = s3_authorization_request(
|
||||
req.method().clone(),
|
||||
req.uri().clone(),
|
||||
&authenticated_zip_download_auth_context(req),
|
||||
req.envelope().server_context().cloned(),
|
||||
req.req_info().cloned(),
|
||||
);
|
||||
authorize_owned_request(&mut authorization, Action::S3Action(action), None).await
|
||||
authorize_request(&mut authorization, Action::S3Action(action))
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Builds the legacy authorization request used when a download token is redeemed.
|
||||
/// The S3 gate (`storage::access::authorize_request`) still evaluates the s3s
|
||||
/// request shape, so the facts it reads are copied into one for the duration
|
||||
/// of the call: the authenticated `ReqInfo`, the peer and proxy-reported client
|
||||
/// addresses, the headers and query the policy conditions come from, and the
|
||||
/// server slot that selects the store. The gate reads neither the presented
|
||||
/// credentials nor the region or service, so those stay unset, and nothing it
|
||||
/// attaches to the request is read afterwards.
|
||||
fn s3_authorization_request(
|
||||
method: Method,
|
||||
uri: Uri,
|
||||
@@ -951,103 +943,6 @@ mod tests {
|
||||
use super::*;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
|
||||
#[tokio::test]
|
||||
async fn zip_scope_authorizes_real_iam_and_restores_prefix_query() {
|
||||
use crate::admin::operation::test_support::TestRequest;
|
||||
use crate::admin::runtime_sources::{AppContext, IamInterface, KmsInterface};
|
||||
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use rustfs_iam::store::{Store as _, UserType};
|
||||
use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq};
|
||||
use rustfs_policy::policy::Policy;
|
||||
|
||||
struct ReadyIam(Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>>);
|
||||
impl IamInterface for ReadyIam {
|
||||
fn handle(&self) -> Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>> {
|
||||
Arc::clone(&self.0)
|
||||
}
|
||||
fn is_ready(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
struct TestKms;
|
||||
impl KmsInterface for TestKms {
|
||||
fn handle(&self) -> Arc<rustfs_kms::KmsServiceManager> {
|
||||
Arc::new(rustfs_kms::KmsServiceManager::new())
|
||||
}
|
||||
}
|
||||
|
||||
let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
store
|
||||
.make_bucket("photos", &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
rustfs_iam::store::object::ObjectStore::new(Arc::clone(&store))
|
||||
.save_iam_config(
|
||||
serde_json::json!({"version": 1}),
|
||||
format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX),
|
||||
)
|
||||
.await
|
||||
.expect("seed IAM format");
|
||||
let iam = rustfs_iam::build_iam_sys(Arc::clone(&store)).await.expect("initialize IAM");
|
||||
let access_key = "zip-scope-user";
|
||||
iam.create_user(
|
||||
access_key,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: "zip-scope-secret-key".to_owned(),
|
||||
policy: None,
|
||||
status: AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create user");
|
||||
iam.set_policy(
|
||||
"zip-scope-policy",
|
||||
Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListBucket","Resource":"arn:aws:s3:::photos","Condition":{"StringEquals":{"s3:prefix":"allowed/"}}},{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/allowed/*"}]}"#)
|
||||
.expect("parse policy"),
|
||||
)
|
||||
.await
|
||||
.expect("store policy");
|
||||
iam.policy_db_set(access_key, UserType::Reg, false, "zip-scope-policy")
|
||||
.await
|
||||
.expect("attach policy");
|
||||
let credentials = iam.get_user(access_key).await.expect("load user").credentials;
|
||||
let context = Arc::new(AppContext::new(store, Arc::new(ReadyIam(iam)), Arc::new(TestKms)));
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let original_uri = "/rustfs/admin/v3/object-zip-downloads?prefix=old&marker=1";
|
||||
let mut request = TestRequest::new(Method::POST, original_uri)
|
||||
.extension(slot)
|
||||
.extension(ReqInfo {
|
||||
cred: Some(credentials),
|
||||
..Default::default()
|
||||
})
|
||||
.build();
|
||||
|
||||
let mut selection = valid_request();
|
||||
selection.prefix = None;
|
||||
selection.prefixes = vec!["allowed/".to_owned()];
|
||||
selection.objects = vec!["allowed/file".to_owned()];
|
||||
authorize_object_zip_download_scope(&mut request, &selection)
|
||||
.await
|
||||
.expect("allowed prefix and object");
|
||||
assert_eq!(request.uri().to_string(), original_uri);
|
||||
|
||||
selection.prefixes = vec!["denied/".to_owned()];
|
||||
let err = authorize_object_zip_download_scope(&mut request, &selection)
|
||||
.await
|
||||
.expect_err("another prefix is denied");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
assert_eq!(request.uri().to_string(), original_uri);
|
||||
|
||||
selection.prefixes = vec!["allowed/".to_owned()];
|
||||
selection.objects = vec!["denied/file".to_owned()];
|
||||
let err = authorize_object_zip_download_scope(&mut request, &selection)
|
||||
.await
|
||||
.expect_err("another object is denied");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
assert_eq!(request.uri().to_string(), original_uri);
|
||||
}
|
||||
|
||||
fn valid_request() -> CreateObjectZipDownloadRequest {
|
||||
CreateObjectZipDownloadRequest {
|
||||
bucket: "photos".to_string(),
|
||||
|
||||
@@ -343,7 +343,7 @@ impl Operation for AddServiceAccount {
|
||||
let (cred, owner) =
|
||||
check_key_valid(get_session_token(req.uri(), req.headers()).unwrap_or_default(), req_cred.access_key()).await?;
|
||||
|
||||
if !req_cred.secret_key_matches(&cred.secret_key) {
|
||||
if !constant_time_eq(&cred.secret_key, req_cred.secret_key()) {
|
||||
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "access denied"));
|
||||
}
|
||||
|
||||
|
||||
@@ -222,7 +222,7 @@ async fn handle_assume_role(
|
||||
|
||||
// The signature was verified before the request body was consumed. A fresh
|
||||
// IAM lookup must not silently replace the secret that authenticated it.
|
||||
if !user.secret_key_matches(&cred.secret_key) {
|
||||
if !crate::auth::constant_time_eq(&cred.secret_key, user.secret_key()) {
|
||||
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use crate::admin::operation::{AdminRequest, AdminResponse};
|
||||
use crate::admin::runtime_sources;
|
||||
use crate::admin::runtime_sources::default_admin_usecase;
|
||||
use crate::admin::storage_api::access::{ReqInfo, authorize_owned_internal_object_request};
|
||||
use crate::admin::storage_api::access::{ReqInfo, authorize_internal_object_request};
|
||||
use crate::admin::storage_api::bucket::metadata::table_catalog_path_hash;
|
||||
use crate::admin::storage_api::runtime::ECStore;
|
||||
use crate::admin::utils::owned_empty_response;
|
||||
@@ -1088,8 +1088,12 @@ fn build_sensitive_json_response<T: Serialize>(status: StatusCode, body: &T) ->
|
||||
Ok(AdminResponse::with_headers(status, body, headers))
|
||||
}
|
||||
|
||||
// These handlers still use the legacy router while their request bodies migrate.
|
||||
// TODO(T1.12-codemod): move them to the owned operation boundary.
|
||||
// The commit-path handlers (`TableCommitObjectBackend::for_request`) authorize
|
||||
// exact object keys through `storage::access::authorize_internal_object_request`,
|
||||
// which still takes the s3s request. Until that path has an owned form they keep
|
||||
// the s3s request and read every other request fact through this owned view.
|
||||
// TODO(T1.12-codemod): remove once the generic object authorization path moves
|
||||
// to the request envelope.
|
||||
type LegacyRequest = s3s::S3Request<s3s::Body>;
|
||||
type LegacyResponse = s3s::S3Response<(StatusCode, s3s::Body)>;
|
||||
|
||||
@@ -1314,14 +1318,15 @@ fn install_table_catalog_s3_request_info(req: &mut LegacyRequest, principal: &Ta
|
||||
}
|
||||
|
||||
async fn authorize_table_catalog_s3_actions(
|
||||
req: &mut AdminRequest,
|
||||
req: &mut LegacyRequest,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
actions: &[S3Action],
|
||||
) -> S3Result<()> {
|
||||
let original = {
|
||||
let req_info = req
|
||||
.req_info_mut()
|
||||
.extensions
|
||||
.get_mut::<ReqInfo>()
|
||||
.ok_or_else(|| s3_error!(AccessDenied, "authentication required"))?;
|
||||
(
|
||||
req_info.bucket.replace(bucket.to_string()),
|
||||
@@ -1332,15 +1337,15 @@ async fn authorize_table_catalog_s3_actions(
|
||||
|
||||
let mut result = Ok(());
|
||||
for action in actions {
|
||||
if let Err(err) = authorize_owned_internal_object_request(req.envelope_mut(), Action::S3Action(*action)).await {
|
||||
if let Err(err) = authorize_internal_object_request(req, Action::S3Action(*action)).await {
|
||||
result = Err(err);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if let Some(req_info) = req.req_info_mut() {
|
||||
if let Some(req_info) = req.extensions.get_mut::<ReqInfo>() {
|
||||
(req_info.bucket, req_info.object, req_info.version_id) = original;
|
||||
}
|
||||
result
|
||||
result.map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn table_catalog_request_principal(req: &AdminRequest) -> S3Result<TableCatalogRequestPrincipal> {
|
||||
@@ -1378,7 +1383,7 @@ async fn table_catalog_request_principal(req: &AdminRequest) -> S3Result<TableCa
|
||||
|
||||
#[derive(Clone)]
|
||||
enum TableCommitObjectAuthorization {
|
||||
Request(Arc<tokio::sync::Mutex<AdminRequest>>),
|
||||
Request(Arc<tokio::sync::Mutex<LegacyRequest>>),
|
||||
Preauthorized,
|
||||
#[cfg(test)]
|
||||
Test {
|
||||
@@ -1454,7 +1459,7 @@ where
|
||||
fn for_request(backend: B, req: LegacyRequest) -> Self {
|
||||
Self::new(
|
||||
backend,
|
||||
TableCommitObjectAuthorization::Request(Arc::new(tokio::sync::Mutex::new(legacy_request_view(&req)))),
|
||||
TableCommitObjectAuthorization::Request(Arc::new(tokio::sync::Mutex::new(req))),
|
||||
rustfs_utils::get_env_bool(crate::table_catalog::ENV_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED, false),
|
||||
)
|
||||
}
|
||||
@@ -2088,7 +2093,7 @@ where
|
||||
let result = match &self.authorization {
|
||||
TableCommitObjectAuthorization::Request(req) => {
|
||||
let req = req.lock().await;
|
||||
authorize_table_warehouse_location_request(&req, table_bucket, location).await
|
||||
authorize_table_warehouse_location_request(&legacy_request_view(&req), table_bucket, location).await
|
||||
}
|
||||
TableCommitObjectAuthorization::Preauthorized => Ok(()),
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -77,7 +77,8 @@ fn table_catalog_handler_request(context: Arc<AppContext>, access_key: &str, sec
|
||||
load_table_request_builder(context, access_key, secret_key).build()
|
||||
}
|
||||
|
||||
/// Builds the legacy request accepted by the commit backend, which derives an owned view.
|
||||
/// The s3s form of [`table_catalog_handler_request`] for the commit backend,
|
||||
/// which still holds the legacy request (see `legacy_request_view`).
|
||||
// TODO(T1.12-codemod): remove with `TableCommitObjectBackend::for_request`'s s3s parameter.
|
||||
fn legacy_table_catalog_handler_request(
|
||||
context: Arc<AppContext>,
|
||||
@@ -1294,10 +1295,6 @@ async fn table_storage_credentials_are_rejected_by_catalog_authentication() {
|
||||
#[tokio::test]
|
||||
async fn warehouse_location_claim_checks_request_identity_and_relocation() {
|
||||
let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
store
|
||||
.make_bucket("warehouse", &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create warehouse bucket");
|
||||
rustfs_iam::store::object::ObjectStore::new(store.clone())
|
||||
.save_iam_config(
|
||||
serde_json::json!({"version": 1}),
|
||||
@@ -1335,18 +1332,6 @@ async fn warehouse_location_claim_checks_request_identity_and_relocation() {
|
||||
iam.policy_db_set(access_key, UserType::Reg, false, "location-creator")
|
||||
.await
|
||||
.expect("attach location policy");
|
||||
iam.set_policy(
|
||||
"owned-object-policy",
|
||||
Policy::parse_config(
|
||||
br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::warehouse/owned/*"},{"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::warehouse/owned/writable"}]}"#,
|
||||
)
|
||||
.expect("parse object policy"),
|
||||
)
|
||||
.await
|
||||
.expect("store object policy");
|
||||
iam.policy_db_set(access_key, UserType::Reg, false, "location-creator,owned-object-policy")
|
||||
.await
|
||||
.expect("attach object policy");
|
||||
let context = Arc::new(AppContext::new(store, Arc::new(RequestIam { handle: iam }), Arc::new(RequestKms)));
|
||||
assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
|
||||
access_key: "location-root".to_string(),
|
||||
@@ -1354,125 +1339,6 @@ async fn warehouse_location_claim_checks_request_identity_and_relocation() {
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
}));
|
||||
// Exercise the actual owned internal authorization path, including its temporary target.
|
||||
let mut legacy = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
let principal = table_catalog_request_principal(&legacy_request_view(&legacy))
|
||||
.await
|
||||
.expect("resolve principal");
|
||||
install_table_catalog_s3_request_info(&mut legacy, &principal).expect("install authorization context");
|
||||
let info = legacy.extensions.get_mut::<ReqInfo>().expect("request info");
|
||||
info.bucket = Some("original-bucket".to_owned());
|
||||
info.object = Some("original-object".to_owned());
|
||||
info.version_id = Some("original-version".to_owned());
|
||||
let mut owned = legacy_request_view(&legacy);
|
||||
for (object, allowed) in [("owned/allowed", true), ("shared/denied", false)] {
|
||||
let result = authorize_table_catalog_s3_actions(&mut owned, "warehouse", object, &[S3Action::GetObjectAction]).await;
|
||||
if allowed {
|
||||
result.expect("the exact owned object should be allowed");
|
||||
} else {
|
||||
assert_eq!(result.expect_err("the other object should be denied").code(), &S3ErrorCode::AccessDenied);
|
||||
}
|
||||
let info = owned.envelope().req_info().expect("request info remains installed");
|
||||
assert_eq!(info.bucket.as_deref(), Some("original-bucket"));
|
||||
assert_eq!(info.object.as_deref(), Some("original-object"));
|
||||
assert_eq!(info.version_id.as_deref(), Some("original-version"));
|
||||
}
|
||||
// The exact-key check selects internal-object mode: a request header the object
|
||||
// write does not apply (a malformed x-amz-tagging) must not reach the decision,
|
||||
// while the ordinary owned entry point still evaluates it and rejects it.
|
||||
let mut tagged = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
install_table_catalog_s3_request_info(&mut tagged, &principal).expect("install authorization context");
|
||||
tagged
|
||||
.headers
|
||||
.insert("x-amz-tagging", http::HeaderValue::from_static("classification=%ZZ"));
|
||||
let mut tagged = legacy_request_view(&tagged);
|
||||
authorize_table_catalog_s3_actions(&mut tagged, "warehouse", "owned/writable", &[S3Action::PutObjectAction])
|
||||
.await
|
||||
.expect("internal-object mode ignores request options the write does not apply");
|
||||
let info = tagged.req_info_mut().expect("request info");
|
||||
info.bucket = Some("warehouse".to_owned());
|
||||
info.object = Some("owned/writable".to_owned());
|
||||
crate::app::authorize_owned_request(
|
||||
tagged.envelope_mut(),
|
||||
rustfs_policy::policy::action::Action::S3Action(S3Action::PutObjectAction),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("ordinary mode evaluates the malformed tagging header");
|
||||
let objects = TestTableCatalogObjectBackend::default();
|
||||
objects
|
||||
.seed_object("warehouse", "owned/read-only", b"readable".to_vec())
|
||||
.await;
|
||||
objects.seed_object("warehouse", "shared/denied", b"unchanged".to_vec()).await;
|
||||
let backend = || {
|
||||
let mut request = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
install_table_catalog_s3_request_info(&mut request, &principal).expect("install authorization context");
|
||||
TableCommitObjectBackend::for_request(objects.clone(), request)
|
||||
};
|
||||
let allowed = backend();
|
||||
let object = allowed
|
||||
.read_object("warehouse", "owned/read-only")
|
||||
.await
|
||||
.expect("authorized read")
|
||||
.expect("seeded object");
|
||||
assert_eq!(object.data, b"readable");
|
||||
allowed
|
||||
.put_object(
|
||||
"warehouse",
|
||||
"owned/writable",
|
||||
b"written".to_vec(),
|
||||
crate::table_catalog::TableCatalogPutPrecondition::Any,
|
||||
)
|
||||
.await
|
||||
.expect("authorized write");
|
||||
assert_eq!(
|
||||
objects
|
||||
.read_object("warehouse", "owned/writable")
|
||||
.await
|
||||
.expect("read stored object")
|
||||
.expect("written object")
|
||||
.data,
|
||||
b"written"
|
||||
);
|
||||
for (object, write) in [("shared/denied", false), ("owned/read-only", true), ("shared/denied", true)] {
|
||||
let denied = backend();
|
||||
let result = if write {
|
||||
denied
|
||||
.put_object(
|
||||
"warehouse",
|
||||
object,
|
||||
b"modified".to_vec(),
|
||||
crate::table_catalog::TableCatalogPutPrecondition::Any,
|
||||
)
|
||||
.await
|
||||
.map(|_| ())
|
||||
} else {
|
||||
denied.read_object("warehouse", object).await.map(|_| ())
|
||||
};
|
||||
assert!(result.is_err(), "unauthorized {object} operation must fail");
|
||||
assert_eq!(
|
||||
denied.finish(Ok(())).await.expect_err("retain authorization error").code(),
|
||||
&S3ErrorCode::AccessDenied
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
objects
|
||||
.read_object("warehouse", "owned/read-only")
|
||||
.await
|
||||
.expect("read object")
|
||||
.expect("seeded object")
|
||||
.data,
|
||||
b"readable"
|
||||
);
|
||||
assert_eq!(
|
||||
objects
|
||||
.read_object("warehouse", "shared/denied")
|
||||
.await
|
||||
.expect("read object")
|
||||
.expect("seeded object")
|
||||
.data,
|
||||
b"unchanged"
|
||||
);
|
||||
let request = || table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
for location in ["s3://warehouse/owned", "s3://warehouse/owned/", "s3://warehouse/owned/child"] {
|
||||
authorize_table_warehouse_location_request(&request(), "warehouse", location)
|
||||
|
||||
@@ -243,10 +243,7 @@ impl Operation for AddUser {
|
||||
let (cred, owner) =
|
||||
check_key_valid(get_session_token(req.uri(), req.headers()).unwrap_or_default(), input_cred.access_key()).await?;
|
||||
// The caller's current secret outlives the body read below.
|
||||
let input_secret = input_cred
|
||||
.require_secret_key()
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, err.to_string()))?
|
||||
.to_owned();
|
||||
let input_secret = input_cred.secret_key().to_owned();
|
||||
|
||||
let ak = query.access_key.as_deref().unwrap_or_default();
|
||||
|
||||
|
||||
@@ -210,10 +210,11 @@ pub(crate) mod test_support {
|
||||
/// token is read from the request like the s3s edge does, so a test
|
||||
/// that sets `x-amz-security-token` keeps working unchanged.
|
||||
pub(crate) fn credentials(mut self, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Self {
|
||||
self.credentials = Some(
|
||||
RequestCredentials::new(access_key.to_owned(), secret_key.to_owned(), session_token.map(str::to_owned))
|
||||
.expect("test credentials carry a non-empty secret"),
|
||||
);
|
||||
self.credentials = Some(RequestCredentials::new(
|
||||
access_key.to_owned(),
|
||||
secret_key.to_owned(),
|
||||
session_token.map(str::to_owned),
|
||||
));
|
||||
self
|
||||
}
|
||||
|
||||
@@ -245,7 +246,11 @@ pub(crate) mod test_support {
|
||||
pub(crate) fn build(self) -> AdminRequest {
|
||||
let credentials = self.credentials.map(|credentials| match credentials.session_token() {
|
||||
Some(_) => credentials,
|
||||
None => credentials.with_session_token(get_session_token(&self.uri, &self.headers).map(str::to_owned)),
|
||||
None => RequestCredentials::new(
|
||||
credentials.access_key().to_owned(),
|
||||
credentials.secret_key().to_owned(),
|
||||
get_session_token(&self.uri, &self.headers).map(str::to_owned),
|
||||
),
|
||||
});
|
||||
let head = RequestHead {
|
||||
method: self.method,
|
||||
@@ -293,7 +298,7 @@ mod tests {
|
||||
.build();
|
||||
let credentials = req.credentials().expect("credentials were supplied");
|
||||
assert_eq!(credentials.access_key(), "AKIATEST");
|
||||
assert_eq!(credentials.secret_key(), Some("secret"));
|
||||
assert_eq!(credentials.secret_key(), "secret");
|
||||
assert_eq!(credentials.session_token(), Some("from-header"));
|
||||
|
||||
let explicit = TestRequest::new(Method::GET, "/rustfs/admin/v3/info")
|
||||
|
||||
@@ -3882,7 +3882,7 @@ mod tests {
|
||||
.credentials()
|
||||
.ok_or_else(|| OwnedS3Error::with_message(OwnedS3ErrorCode::AccessDenied, "owned admin identity is required"))?;
|
||||
let identity_matches = credentials.access_key() == "AKIAOWNED"
|
||||
&& credentials.secret_key_matches("owned-secret")
|
||||
&& constant_time_eq(credentials.secret_key(), "owned-secret")
|
||||
&& credentials
|
||||
.session_token()
|
||||
.is_some_and(|token| constant_time_eq(token, "owned-session"));
|
||||
|
||||
@@ -953,8 +953,9 @@ pub(crate) mod data_usage {
|
||||
}
|
||||
|
||||
pub(crate) mod access {
|
||||
pub(crate) use crate::app::{authorize_owned_internal_object_request, authorize_owned_request};
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{ReqInfo, authorize_request};
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{
|
||||
ReqInfo, authorize_internal_object_request, authorize_request,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::request_context_consumer::{RequestContext, spawn_traced};
|
||||
}
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
|
||||
#[cfg(test)]
|
||||
use super::storage_api::bucket_usecase::access::ReqInfo;
|
||||
use super::storage_api::bucket_usecase::access::{
|
||||
ListBucketsAuthzRequest, TableDataPlaneListAccess, authorize_request, bucket_config_mutation_incarnation,
|
||||
log_list_buckets_iam_implicit_deny, prepare_list_buckets_iam_authorization, prepare_odm_read_generation, req_info_ref,
|
||||
TableDataPlaneListAccess, authorize_request, bucket_config_mutation_incarnation, log_list_buckets_iam_implicit_deny,
|
||||
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, req_info_ref,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::storage_api::bucket_usecase::bucket::target::BucketTarget;
|
||||
@@ -47,7 +47,7 @@ use super::storage_api::bucket_usecase::bucket::{
|
||||
versioning_sys::BucketVersioningSys,
|
||||
};
|
||||
use super::storage_api::bucket_usecase::contract::bucket::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
|
||||
BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
|
||||
};
|
||||
use super::storage_api::bucket_usecase::contract::list::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _,
|
||||
@@ -56,9 +56,9 @@ use super::storage_api::bucket_usecase::error::StorageError;
|
||||
use super::storage_api::bucket_usecase::helper::{OperationHelper, spawn_background_with_context};
|
||||
use super::storage_api::bucket_usecase::object_utils::to_s3s_xml_etag;
|
||||
use super::storage_api::bucket_usecase::s3_api::bucket::{
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_object_versions_output, build_list_objects_output,
|
||||
build_list_objects_v2_output, encode_list_objects_v2_continuation_token, parse_list_object_versions_params,
|
||||
parse_list_objects_v2_params, rustfs_owner,
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
|
||||
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
|
||||
parse_list_object_versions_params, parse_list_objects_v2_params, rustfs_owner,
|
||||
};
|
||||
use super::storage_api::bucket_usecase::{
|
||||
get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
@@ -66,7 +66,6 @@ use super::storage_api::bucket_usecase::{
|
||||
};
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
use super::storage_api::diagnostic;
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::bucket_list_through as list_through;
|
||||
use crate::app::object::shared::mark_on_demand_migration_list_local_only;
|
||||
use crate::app::object_data_cache::invalidate_object_data_cache_bucket_after_delete;
|
||||
@@ -88,14 +87,12 @@ use http::StatusCode;
|
||||
use metrics::counter;
|
||||
use rustfs_config::RUSTFS_REGION;
|
||||
use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration, PersistedTagging};
|
||||
use rustfs_gateway_types::dto as gateway_dto;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedBucketEncryptionConfiguration, PersistedNotificationConfiguration,
|
||||
PersistedObjectLockConfiguration as ObjectLockConfiguration, PersistedPublicAccessBlockConfiguration,
|
||||
PersistedVersioningConfiguration,
|
||||
};
|
||||
use rustfs_gateway_types::persistence::{PersistedLifecycleConfiguration, PersistedReplicationConfiguration};
|
||||
use rustfs_gateway_types::{BucketName, Timestamp as GatewayTimestamp};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
|
||||
use rustfs_policy::policy::{
|
||||
@@ -119,15 +116,15 @@ use s3s::dto::{
|
||||
GetBucketCorsInput, GetBucketEncryptionInput, GetBucketLifecycleConfigurationInput, GetBucketLocationInput,
|
||||
GetBucketLocationOutput, GetBucketNotificationConfigurationInput, GetBucketPolicyInput, GetBucketPolicyOutput,
|
||||
GetBucketPolicyStatusInput, GetBucketPolicyStatusOutput, GetBucketReplicationInput, GetBucketTaggingInput,
|
||||
GetBucketVersioningInput, GetPublicAccessBlockInput, HeadBucketInput, HeadBucketOutput, LifecycleRule,
|
||||
ListObjectVersionsInput, ListObjectVersionsOutput, ListObjectsInput, ListObjectsOutput, ListObjectsV2Input,
|
||||
ListObjectsV2Output, MetadataEntry, NotificationConfiguration, NotificationConfigurationFilter, Object, ObjectStorageClass,
|
||||
ObjectVersion, ObjectVersionStorageClass, PolicyStatus, PutBucketCorsInput, PutBucketCorsOutput, PutBucketEncryptionInput,
|
||||
PutBucketEncryptionOutput, PutBucketLifecycleConfigurationInput, PutBucketLifecycleConfigurationOutput,
|
||||
PutBucketNotificationConfigurationInput, PutBucketNotificationConfigurationOutput, PutBucketPolicyInput,
|
||||
PutBucketPolicyOutput, PutBucketReplicationInput, PutBucketReplicationOutput, PutBucketTaggingInput, PutBucketTaggingOutput,
|
||||
PutBucketVersioningInput, PutBucketVersioningOutput, PutPublicAccessBlockInput, PutPublicAccessBlockOutput,
|
||||
ServerSideEncryption, Timestamp, UserMetadata,
|
||||
GetBucketVersioningInput, GetPublicAccessBlockInput, HeadBucketInput, HeadBucketOutput, LifecycleRule, ListBucketsInput,
|
||||
ListBucketsOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ListObjectsInput, ListObjectsOutput,
|
||||
ListObjectsV2Input, ListObjectsV2Output, MetadataEntry, NotificationConfiguration, NotificationConfigurationFilter, Object,
|
||||
ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, PolicyStatus, PutBucketCorsInput, PutBucketCorsOutput,
|
||||
PutBucketEncryptionInput, PutBucketEncryptionOutput, PutBucketLifecycleConfigurationInput,
|
||||
PutBucketLifecycleConfigurationOutput, PutBucketNotificationConfigurationInput, PutBucketNotificationConfigurationOutput,
|
||||
PutBucketPolicyInput, PutBucketPolicyOutput, PutBucketReplicationInput, PutBucketReplicationOutput, PutBucketTaggingInput,
|
||||
PutBucketTaggingOutput, PutBucketVersioningInput, PutBucketVersioningOutput, PutPublicAccessBlockInput,
|
||||
PutPublicAccessBlockOutput, ServerSideEncryption, Timestamp, UserMetadata,
|
||||
};
|
||||
use s3s::region::Region;
|
||||
use s3s::xml;
|
||||
@@ -1406,40 +1403,6 @@ fn lifecycle_has_abort_multipart_rules(config: &PersistedLifecycleConfiguration)
|
||||
.any(|rule| rule.status == "Enabled" && rule.abort_incomplete_multipart_upload.is_some())
|
||||
}
|
||||
|
||||
/// The ListBuckets answer for `bucket_infos`, in the gateway's types.
|
||||
///
|
||||
/// A stored name the S3 bucket grammar refuses cannot be written into the answer and is left out.
|
||||
/// The storage inventory holds such names only for RustFS-internal buckets (the SFTP host-key
|
||||
/// store); `rustfs/src/protocols/client.rs` applies the same rule to its listings.
|
||||
fn list_buckets_output(bucket_infos: &[BucketInfo]) -> gateway_dto::ListBucketsOutput {
|
||||
let buckets = bucket_infos
|
||||
.iter()
|
||||
.filter_map(|bucket_info| {
|
||||
let name = BucketName::new(bucket_info.name.clone()).ok()?;
|
||||
let created = bucket_info.created.unwrap_or(time::OffsetDateTime::UNIX_EPOCH);
|
||||
let seconds = created.unix_timestamp();
|
||||
let creation_date =
|
||||
GatewayTimestamp::from_secs_nanos(seconds, created.nanosecond()).unwrap_or(GatewayTimestamp::from_secs(seconds));
|
||||
Some(gateway_dto::Bucket {
|
||||
name,
|
||||
creation_date,
|
||||
..Default::default()
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let owner = rustfs_owner();
|
||||
|
||||
gateway_dto::ListBucketsOutput {
|
||||
buckets,
|
||||
owner: gateway_dto::Owner {
|
||||
id: owner.id,
|
||||
display_name: owner.display_name,
|
||||
},
|
||||
continuation_token: None,
|
||||
prefix: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct DefaultBucketUsecase {
|
||||
context: Option<Arc<AppContext>>,
|
||||
@@ -1776,61 +1739,21 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(GetBucketLocationOutput { location_constraint }))
|
||||
}
|
||||
|
||||
/// Lists the buckets the caller of `envelope` may see, for either edge.
|
||||
///
|
||||
/// The access phase must have installed the caller's `ReqInfo` in `envelope`. A caller allowed
|
||||
/// `s3:ListAllMyBuckets` sees every bucket; any other caller sees the buckets it may list or
|
||||
/// locate, and is refused when there is none. `_input` is accepted and not read: RustFS answers
|
||||
/// one unpaginated listing and ignores `prefix`, `max-buckets`, `continuation-token` and
|
||||
/// `bucket-region`, as it always has.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// `AccessDenied` for an anonymous caller and for one that may see no bucket; `InternalError`
|
||||
/// when the store or IAM is not initialized or the access phase did not run; otherwise the
|
||||
/// storage error of the listing.
|
||||
#[instrument(level = "debug", skip(self, envelope, _input))]
|
||||
pub async fn execute_list_buckets(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
_input: gateway_dto::ListBucketsInput,
|
||||
) -> rustfs_s3_types::S3Result<gateway_dto::ListBucketsOutput> {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_list_buckets(&self, req: S3Request<ListBucketsInput>) -> S3Result<S3Response<ListBucketsOutput>> {
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(rustfs_s3_types::S3Error::with_message(
|
||||
rustfs_s3_types::S3ErrorCode::InternalError,
|
||||
"Not init",
|
||||
));
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
if envelope
|
||||
.credentials()
|
||||
.is_none_or(|credentials| credentials.access_key().is_empty())
|
||||
{
|
||||
return Err(rustfs_s3_types::S3Error::with_message(
|
||||
rustfs_s3_types::S3ErrorCode::AccessDenied,
|
||||
"Access Denied",
|
||||
));
|
||||
if req.credentials.as_ref().is_none_or(|cred| cred.access_key.is_empty()) {
|
||||
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
|
||||
}
|
||||
|
||||
let Some(req_info) = envelope.req_info() else {
|
||||
return Err(rustfs_s3_types::S3Error::with_message(
|
||||
rustfs_s3_types::S3ErrorCode::InternalError,
|
||||
"ReqInfo not found in request envelope",
|
||||
));
|
||||
};
|
||||
let iam_authorization = prepare_list_buckets_iam_authorization(ListBucketsAuthzRequest {
|
||||
req_info,
|
||||
headers: envelope.headers(),
|
||||
raw_query: envelope.uri().query(),
|
||||
remote_addr: envelope.remote_addr(),
|
||||
client_info: envelope.client_info(),
|
||||
server_context: envelope.server_context(),
|
||||
})
|
||||
.await?;
|
||||
let iam_authorization = prepare_list_buckets_iam_authorization(&req).await?;
|
||||
let bucket_infos = if iam_authorization.is_allowed("", S3Action::ListAllMyBucketsAction).await {
|
||||
store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?
|
||||
} else {
|
||||
log_list_buckets_iam_implicit_deny(req_info);
|
||||
log_list_buckets_iam_implicit_deny(&req)?;
|
||||
let bucket_infos = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?;
|
||||
let mut visible_bucket_infos = Vec::new();
|
||||
for info in bucket_infos {
|
||||
@@ -1849,7 +1772,7 @@ impl DefaultBucketUsecase {
|
||||
visible_bucket_infos
|
||||
};
|
||||
|
||||
Ok(list_buckets_output(&bucket_infos))
|
||||
Ok(S3Response::new(build_list_buckets_output(&bucket_infos)))
|
||||
}
|
||||
|
||||
pub async fn execute_delete_bucket_encryption(
|
||||
@@ -3989,16 +3912,36 @@ mod tests {
|
||||
req
|
||||
}
|
||||
|
||||
fn usecase_method_source<'a>(source: &'a str, method: &str) -> &'a str {
|
||||
let public_start_marker = format!("pub async fn {method}");
|
||||
let private_start_marker = format!("async fn {method}");
|
||||
let (start, marker_len) = if let Some(start) = source.find(&public_start_marker) {
|
||||
(start, public_start_marker.len())
|
||||
/// Bucket usecase source: this file plus `bucket_usecase/`, where a split moves its methods.
|
||||
fn bucket_usecase_sources() -> crate::guard_sources::SourceSet {
|
||||
crate::guard_sources::SourceSet::read("src/app/bucket_usecase.rs", "src/app/bucket_usecase")
|
||||
}
|
||||
|
||||
/// Text after each exact `{marker}(` or `{marker}<` definition, in set order.
|
||||
fn method_definitions<'a>(source: &'a crate::guard_sources::SourceSet, marker: &str) -> Vec<&'a str> {
|
||||
source
|
||||
.files()
|
||||
.iter()
|
||||
.flat_map(|file| {
|
||||
file.text
|
||||
.match_indices(marker)
|
||||
.map(move |(start, _)| &file.text[start + marker.len()..])
|
||||
.filter(|rest| rest.starts_with(['(', '<']))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The body of the one definition of `method` across the bucket usecase set, preferring the
|
||||
/// public form; it ends at the next method of the same indentation or at the end of its file.
|
||||
fn usecase_method_source<'a>(source: &'a crate::guard_sources::SourceSet, method: &str) -> &'a str {
|
||||
let public = method_definitions(source, &format!("pub async fn {method}"));
|
||||
let definitions = if public.is_empty() {
|
||||
method_definitions(source, &format!("async fn {method}"))
|
||||
} else {
|
||||
let start = source.find(&private_start_marker).expect("method should exist");
|
||||
(start, private_start_marker.len())
|
||||
public
|
||||
};
|
||||
let rest = &source[start + marker_len..];
|
||||
assert_eq!(definitions.len(), 1, "{method} must be defined exactly once across {}", source.describe());
|
||||
let rest = definitions[0];
|
||||
let end = [rest.find("\n pub async fn "), rest.find("\n async fn ")]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
@@ -4009,7 +3952,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn bucket_create_and_delete_use_admitted_fresh_tasks() {
|
||||
let source = include_str!("bucket_usecase.rs");
|
||||
let sources = bucket_usecase_sources();
|
||||
let source = &sources;
|
||||
for (method, operation, inner_call) in [
|
||||
(
|
||||
"execute_create_bucket",
|
||||
@@ -4034,7 +3978,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn bucket_metadata_config_changes_notify_peer_metadata_reload() {
|
||||
let source = include_str!("bucket_usecase.rs");
|
||||
let sources = bucket_usecase_sources();
|
||||
let source = &sources;
|
||||
for (method, operation, scanner_maintenance_change) in [
|
||||
("execute_delete_bucket_policy", "delete bucket policy", false),
|
||||
("execute_put_bucket_policy", "put bucket policy", false),
|
||||
@@ -5161,60 +5106,13 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_buckets_returns_internal_error_when_store_uninitialized() {
|
||||
let req = build_request((), Method::GET);
|
||||
let input = ListBucketsInput::builder().build().unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_list_buckets(RequestEnvelope::from_s3s(&req), gateway_dto::ListBucketsInput::default())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
fn bucket_info(name: &str, created: Option<time::OffsetDateTime>) -> BucketInfo {
|
||||
BucketInfo {
|
||||
name: name.to_owned(),
|
||||
created,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_buckets_output_carries_names_dates_and_the_owner() {
|
||||
let created = time::OffsetDateTime::from_unix_timestamp_nanos(1_700_000_000_123_000_000).expect("valid timestamp");
|
||||
|
||||
let output = list_buckets_output(&[bucket_info("alpha", Some(created)), bucket_info("beta", None)]);
|
||||
|
||||
let listed: Vec<(&str, i64, u32)> = output
|
||||
.buckets
|
||||
.iter()
|
||||
.map(|bucket| (bucket.name.as_str(), bucket.creation_date.secs(), bucket.creation_date.subsec_nanos()))
|
||||
.collect();
|
||||
assert_eq!(listed, [("alpha", 1_700_000_000, 123_000_000), ("beta", 0, 0)]);
|
||||
assert!(output.owner.id.is_some(), "the listing names its owner");
|
||||
assert_eq!(output.continuation_token, None);
|
||||
assert_eq!(output.prefix, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_buckets_output_leaves_out_names_the_bucket_grammar_refuses() {
|
||||
let output = list_buckets_output(&[
|
||||
bucket_info("sftp_host_keys", None),
|
||||
bucket_info("UPPER", None),
|
||||
bucket_info("ab", None),
|
||||
bucket_info("kept", None),
|
||||
]);
|
||||
|
||||
let names: Vec<&str> = output.buckets.iter().map(|bucket| bucket.name.as_str()).collect();
|
||||
assert_eq!(names, ["kept"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_buckets_output_of_no_bucket_is_an_empty_listing_with_an_owner() {
|
||||
let output = list_buckets_output(&[]);
|
||||
|
||||
assert!(output.buckets.is_empty());
|
||||
assert!(output.owner.id.is_some());
|
||||
let err = usecase.execute_list_buckets(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -26,7 +26,6 @@ mod startup;
|
||||
|
||||
pub(crate) use envelope::RequestHead;
|
||||
pub use envelope::{RequestCredentials, RequestEnvelope};
|
||||
pub(crate) use envelope::{authorize_owned_internal_object_request, authorize_owned_request};
|
||||
pub use global::*;
|
||||
pub use handles::*;
|
||||
pub use interfaces::*;
|
||||
|
||||
@@ -29,19 +29,13 @@ use super::ServerContextSlot;
|
||||
use crate::app::metadata_route::WebsiteRead;
|
||||
use crate::app::object::request_body::BodyReadControl;
|
||||
use crate::app::storage_api::context::{
|
||||
AuditRequestView, AuthorizationRequest, DenialAuditor, DenialScope, authorize_request_core,
|
||||
};
|
||||
use crate::app::storage_api::context::{
|
||||
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, MultipartStagingOperation,
|
||||
ObjectTagConditions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectPolicyHeaders,
|
||||
PostObjectRequestMarker, ReqInfo, RequestContext, RequestedCopySource, RequestedObjectLegalHold,
|
||||
RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags, StagedMultipartPartAuthorization,
|
||||
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, ObjectTagConditions,
|
||||
OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectRequestMarker, ReqInfo, RequestContext,
|
||||
TableDataPlaneListAccess, TableDataPlanePublicationGuards,
|
||||
};
|
||||
use crate::auth::{VerifiedPresignedRequest, VerifiedSigV4Request};
|
||||
use crate::shared_types::{NativeTlsTransport, RemoteAddr};
|
||||
use http::{Extensions, HeaderMap, Method, Uri};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::fmt;
|
||||
use std::net::SocketAddr;
|
||||
@@ -51,109 +45,42 @@ use zeroize::Zeroizing;
|
||||
/// The identity a request was authenticated with.
|
||||
///
|
||||
/// The secret key and the session token are bearer material: the type has no
|
||||
/// `PartialEq`, and its `Debug` output redacts both. An edge that keeps the
|
||||
/// caller's secret (it verified the signature itself) builds the value with
|
||||
/// [`Self::without_secret_key`]; the absence is then `None`, never a
|
||||
/// substitute string a reader could compare, seal or derive with.
|
||||
/// `PartialEq`, and its `Debug` output redacts both.
|
||||
#[derive(Clone)]
|
||||
pub struct RequestCredentials {
|
||||
access_key: String,
|
||||
secret_key: Option<Zeroizing<String>>,
|
||||
secret_key: Zeroizing<String>,
|
||||
session_token: Option<Zeroizing<String>>,
|
||||
}
|
||||
|
||||
/// An edge handed over an empty secret key. Absence is expressed with
|
||||
/// [`RequestCredentials::without_secret_key`], so an empty secret is refused
|
||||
/// rather than read as "absent" or used as a key.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct EmptySecretKey;
|
||||
|
||||
impl fmt::Display for EmptySecretKey {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("handed-over secret key is empty")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for EmptySecretKey {}
|
||||
|
||||
impl RequestCredentials {
|
||||
/// Credentials whose secret the edge handed over.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`EmptySecretKey`] when `secret_key` is empty.
|
||||
pub(crate) fn new(access_key: String, secret_key: String, session_token: Option<String>) -> Result<Self, EmptySecretKey> {
|
||||
if secret_key.is_empty() {
|
||||
return Err(EmptySecretKey);
|
||||
}
|
||||
Ok(Self {
|
||||
access_key,
|
||||
secret_key: Some(Zeroizing::new(secret_key)),
|
||||
session_token: session_token.map(Zeroizing::new),
|
||||
})
|
||||
}
|
||||
|
||||
/// Credentials whose secret the edge kept.
|
||||
pub(crate) fn without_secret_key(access_key: String, session_token: Option<String>) -> Self {
|
||||
pub(crate) fn new(access_key: String, secret_key: String, session_token: Option<String>) -> Self {
|
||||
Self {
|
||||
access_key,
|
||||
secret_key: None,
|
||||
secret_key: Zeroizing::new(secret_key),
|
||||
session_token: session_token.map(Zeroizing::new),
|
||||
}
|
||||
}
|
||||
|
||||
/// The same identity with `session_token` as its token.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_session_token(mut self, session_token: Option<String>) -> Self {
|
||||
self.session_token = session_token.map(Zeroizing::new);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn access_key(&self) -> &str {
|
||||
&self.access_key
|
||||
}
|
||||
|
||||
/// The caller's secret, or `None` when the edge did not hand it over.
|
||||
/// A reader must fail closed on `None`. Never log this value or compare it
|
||||
/// with `==`.
|
||||
pub fn secret_key(&self) -> Option<&str> {
|
||||
self.secret_key.as_deref().map(String::as_str)
|
||||
/// Never log this value or compare it with `==`.
|
||||
pub fn secret_key(&self) -> &str {
|
||||
&self.secret_key
|
||||
}
|
||||
|
||||
pub fn session_token(&self) -> Option<&str> {
|
||||
self.session_token.as_deref().map(String::as_str)
|
||||
}
|
||||
|
||||
/// The handed-over secret, or [`MissingSecretKey`] when the edge kept it.
|
||||
pub(crate) fn require_secret_key(&self) -> Result<&str, MissingSecretKey> {
|
||||
self.secret_key().ok_or(MissingSecretKey)
|
||||
}
|
||||
|
||||
/// Whether the handed-over secret equals `stored`, compared in constant
|
||||
/// time. An absent secret matches nothing, not even an empty `stored`.
|
||||
pub(crate) fn secret_key_matches(&self, stored: &str) -> bool {
|
||||
self.secret_key()
|
||||
.is_some_and(|secret| crate::auth::constant_time_eq(secret, stored))
|
||||
}
|
||||
}
|
||||
|
||||
/// A reader needed the caller's secret, but the edge did not hand it over.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct MissingSecretKey;
|
||||
|
||||
impl fmt::Display for MissingSecretKey {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("the caller's secret key was not handed over for this request")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for MissingSecretKey {}
|
||||
|
||||
impl fmt::Debug for RequestCredentials {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("RequestCredentials")
|
||||
.field("access_key", &self.access_key)
|
||||
.field("secret_key", &self.secret_key.as_ref().map(|_| "<redacted>"))
|
||||
.field("secret_key", &"<redacted>")
|
||||
.field("session_token", &self.session_token.as_ref().map(|_| "<redacted>"))
|
||||
.finish()
|
||||
}
|
||||
@@ -201,7 +128,6 @@ pub struct RequestEnvelope {
|
||||
verified_presigned: Option<VerifiedPresignedRequest>,
|
||||
website_read: Option<WebsiteRead>,
|
||||
post_object: Option<PostObjectRequestMarker>,
|
||||
post_object_policy_headers: Option<PostObjectPolicyHeaders>,
|
||||
// Attached by the authorization phase (`storage/access.rs`) for the use case.
|
||||
req_info: Option<ReqInfo>,
|
||||
bucket_generation: Option<BucketGenerationGuard>,
|
||||
@@ -212,13 +138,6 @@ pub struct RequestEnvelope {
|
||||
table_publication_guards: Option<TableDataPlanePublicationGuards>,
|
||||
table_list_access: Option<TableDataPlaneListAccess>,
|
||||
object_tag_conditions: Option<ObjectTagConditions>,
|
||||
multipart_staging_operation: Option<MultipartStagingOperation>,
|
||||
staged_multipart_part_authorization: Option<StagedMultipartPartAuthorization>,
|
||||
requested_object_lock_retention: Option<RequestedObjectLockRetention>,
|
||||
requested_object_lock_retain_until: Option<RequestedObjectLockRetainUntil>,
|
||||
requested_object_legal_hold: Option<RequestedObjectLegalHold>,
|
||||
requested_object_tags: Option<RequestedObjectTags>,
|
||||
requested_copy_source: Option<RequestedCopySource>,
|
||||
residual_extensions: Extensions,
|
||||
}
|
||||
|
||||
@@ -248,7 +167,6 @@ impl RequestEnvelope {
|
||||
verified_presigned: extensions.remove(),
|
||||
website_read: extensions.remove(),
|
||||
post_object: extensions.remove(),
|
||||
post_object_policy_headers: extensions.remove(),
|
||||
req_info: extensions.remove(),
|
||||
bucket_generation: extensions.remove(),
|
||||
pending_delete_bucket_generation: extensions.remove(),
|
||||
@@ -258,13 +176,6 @@ impl RequestEnvelope {
|
||||
table_publication_guards: extensions.remove(),
|
||||
table_list_access: extensions.remove(),
|
||||
object_tag_conditions: extensions.remove(),
|
||||
multipart_staging_operation: extensions.remove(),
|
||||
staged_multipart_part_authorization: extensions.remove(),
|
||||
requested_object_lock_retention: extensions.remove(),
|
||||
requested_object_lock_retain_until: extensions.remove(),
|
||||
requested_object_legal_hold: extensions.remove(),
|
||||
requested_object_tags: extensions.remove(),
|
||||
requested_copy_source: extensions.remove(),
|
||||
residual_extensions: extensions,
|
||||
}
|
||||
}
|
||||
@@ -401,10 +312,6 @@ impl RequestEnvelope {
|
||||
self.table_publication_guards.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn install_table_publication_guards(&mut self, guards: TableDataPlanePublicationGuards) {
|
||||
self.table_publication_guards = Some(guards);
|
||||
}
|
||||
|
||||
pub(crate) fn table_list_access(&self) -> Option<&TableDataPlaneListAccess> {
|
||||
self.table_list_access.as_ref()
|
||||
}
|
||||
@@ -412,337 +319,19 @@ impl RequestEnvelope {
|
||||
pub(crate) fn object_tag_conditions(&self) -> Option<&ObjectTagConditions> {
|
||||
self.object_tag_conditions.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn install_object_tag_conditions(&mut self, conditions: ObjectTagConditions) {
|
||||
self.object_tag_conditions = Some(conditions);
|
||||
}
|
||||
|
||||
pub(crate) fn multipart_staging_operation(&self) -> Option<&MultipartStagingOperation> {
|
||||
self.multipart_staging_operation.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn staged_multipart_part_authorization(&self) -> Option<&StagedMultipartPartAuthorization> {
|
||||
self.staged_multipart_part_authorization.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn requested_object_lock_retention(&self) -> Option<&RequestedObjectLockRetention> {
|
||||
self.requested_object_lock_retention.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn requested_object_lock_retain_until(&self) -> Option<&RequestedObjectLockRetainUntil> {
|
||||
self.requested_object_lock_retain_until.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn post_object_policy_headers(&self) -> Option<&PostObjectPolicyHeaders> {
|
||||
self.post_object_policy_headers.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn requested_object_legal_hold(&self) -> Option<&RequestedObjectLegalHold> {
|
||||
self.requested_object_legal_hold.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn requested_object_tags(&self) -> Option<&RequestedObjectTags> {
|
||||
self.requested_object_tags.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn requested_copy_source(&self) -> Option<&RequestedCopySource> {
|
||||
self.requested_copy_source.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
/// The authorization core in `storage` reads requests only through
|
||||
/// [`AuthorizationRequest`]; this impl hands it the envelope's named fields.
|
||||
impl AuthorizationRequest for RequestEnvelope {
|
||||
fn headers(&self) -> &HeaderMap {
|
||||
self.headers()
|
||||
}
|
||||
fn uri(&self) -> &http::Uri {
|
||||
self.uri()
|
||||
}
|
||||
fn remote_addr(&self) -> Option<std::net::SocketAddr> {
|
||||
self.remote_addr()
|
||||
}
|
||||
fn client_info(&self) -> Option<&ClientInfo> {
|
||||
self.client_info()
|
||||
}
|
||||
fn server_context(&self) -> Option<&Arc<ServerContextSlot>> {
|
||||
self.server_context()
|
||||
}
|
||||
fn req_info(&self) -> Option<&ReqInfo> {
|
||||
self.req_info()
|
||||
}
|
||||
fn internal_object_authorization(&self) -> bool {
|
||||
false
|
||||
}
|
||||
fn multipart_staging(&self) -> bool {
|
||||
self.multipart_staging_operation().is_some()
|
||||
}
|
||||
fn staged_multipart_part(&self) -> bool {
|
||||
self.staged_multipart_part_authorization().is_some()
|
||||
}
|
||||
fn requested_retention(&self) -> Option<&RequestedObjectLockRetention> {
|
||||
self.requested_object_lock_retention()
|
||||
}
|
||||
fn requested_retain_until(&self) -> Option<&RequestedObjectLockRetainUntil> {
|
||||
self.requested_object_lock_retain_until()
|
||||
}
|
||||
fn requested_legal_hold(&self) -> Option<&RequestedObjectLegalHold> {
|
||||
self.requested_object_legal_hold()
|
||||
}
|
||||
fn requested_object_tags(&self) -> Option<&RequestedObjectTags> {
|
||||
self.requested_object_tags()
|
||||
}
|
||||
fn requested_copy_source(&self) -> Option<&RequestedCopySource> {
|
||||
self.requested_copy_source()
|
||||
}
|
||||
fn policy_headers(&self) -> &HeaderMap {
|
||||
self.post_object_policy_headers().map_or(self.headers(), |headers| &headers.0)
|
||||
}
|
||||
fn object_tag_conditions(&self) -> Option<&ObjectTagConditions> {
|
||||
self.object_tag_conditions()
|
||||
}
|
||||
fn install_object_tag_conditions(&mut self, conditions: ObjectTagConditions) {
|
||||
self.install_object_tag_conditions(conditions);
|
||||
}
|
||||
fn publication_guards(&self) -> Option<&TableDataPlanePublicationGuards> {
|
||||
self.table_publication_guards()
|
||||
}
|
||||
fn install_publication_guards(&mut self, guards: TableDataPlanePublicationGuards) {
|
||||
self.install_table_publication_guards(guards);
|
||||
}
|
||||
fn audit_denial(&self, auditor: &DenialAuditor, action: &Action, reason: &'static str) {
|
||||
auditor.audit_owned(&self.audit_view(), action, reason, DenialScope::Request);
|
||||
}
|
||||
}
|
||||
|
||||
/// Authorizes an owned request through the same IAM, bucket and table policies as the legacy edge.
|
||||
///
|
||||
/// Like the legacy `authorize_request`, it first restores the signed `x-amz-meta-*`
|
||||
/// query parameters into headers, so policy conditions and duplicate rejection see
|
||||
/// the same request shape on both edges.
|
||||
///
|
||||
/// This is only the policy decision. It does **not** perform the composite checks
|
||||
/// the legacy `S3Access` hooks run around their decision, such as the
|
||||
/// `s3:BypassGovernanceRetention` authorization for a governance-bypass header or
|
||||
/// the copy-source read authorization; a caller that needs them runs them itself.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// `InvalidArgument` when the signed query repeats a user metadata key;
|
||||
/// `AccessDenied` when the decision for `action` depends on a request-body input
|
||||
/// the envelope does not carry (see [`require_body_policy_inputs`]) or when the
|
||||
/// policies deny the request.
|
||||
pub(crate) async fn authorize_owned_request(
|
||||
req: &mut RequestEnvelope,
|
||||
action: Action,
|
||||
location_constraint: Option<String>,
|
||||
) -> rustfs_s3_types::S3Result<()> {
|
||||
prepare_owned_authorization(req, action)?;
|
||||
authorize_request_core(req, action, location_constraint, false)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Authorizes one exact object key after its enclosing table-catalog operation has been authorized.
|
||||
///
|
||||
/// It applies the same preparation and has the same limits as [`authorize_owned_request`].
|
||||
pub(crate) async fn authorize_owned_internal_object_request(
|
||||
req: &mut RequestEnvelope,
|
||||
action: Action,
|
||||
) -> rustfs_s3_types::S3Result<()> {
|
||||
prepare_owned_authorization(req, action)?;
|
||||
authorize_request_core(req, action, None, true).await.map_err(Into::into)
|
||||
}
|
||||
|
||||
fn prepare_owned_authorization(req: &mut RequestEnvelope, action: Action) -> rustfs_s3_types::S3Result<()> {
|
||||
crate::auth::hoist_signed_query_params(&mut req.headers, req.uri.query()).map_err(rustfs_s3_types::S3Error::from)?;
|
||||
require_body_policy_inputs(req, action)
|
||||
}
|
||||
|
||||
/// Fails closed when the decision for `action` depends on a value the request
|
||||
/// body supplies but the envelope does not carry it. Policy conditions would
|
||||
/// otherwise fall back to client headers, which the stored value need not match
|
||||
/// (header `OFF` with body `ON` would pass a legal-hold Deny).
|
||||
fn require_body_policy_inputs(req: &RequestEnvelope, action: Action) -> rustfs_s3_types::S3Result<()> {
|
||||
let missing = match action {
|
||||
Action::S3Action(S3Action::PutObjectLegalHoldAction) => req.requested_object_legal_hold.is_none(),
|
||||
Action::S3Action(S3Action::PutObjectTaggingAction | S3Action::PutObjectVersionTaggingAction) => {
|
||||
req.requested_object_tags.is_none()
|
||||
}
|
||||
Action::S3Action(S3Action::PutObjectRetentionAction) => req.requested_object_lock_retention.is_none(),
|
||||
_ => false,
|
||||
} || (req.post_object.is_some() && req.post_object_policy_headers.is_none())
|
||||
|| (req.headers.contains_key(COPY_SOURCE_HEADER) && req.requested_copy_source.is_none());
|
||||
if missing {
|
||||
return Err(rustfs_s3_types::s3_error!(
|
||||
AccessDenied,
|
||||
"request body policy input is missing for this action"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const COPY_SOURCE_HEADER: &str = "x-amz-copy-source";
|
||||
|
||||
impl RequestEnvelope {
|
||||
/// The request fields an audit entry for this request is built from.
|
||||
pub(crate) fn audit_view(&self) -> AuditRequestView<'_> {
|
||||
AuditRequestView::from_facts(
|
||||
self.headers(),
|
||||
self.uri(),
|
||||
self.req_info(),
|
||||
self.client_info(),
|
||||
self.remote_addr(),
|
||||
self.request_context(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn authorization_payloads_move_from_extensions_into_named_fields() {
|
||||
let mut extensions = Extensions::new();
|
||||
extensions.insert(MultipartStagingOperation);
|
||||
extensions.insert(StagedMultipartPartAuthorization);
|
||||
let mut retention = RequestedObjectLockRetention(Default::default());
|
||||
retention.0.mode = Some("COMPLIANCE".to_owned().into());
|
||||
extensions.insert(retention);
|
||||
let retain_until = time::OffsetDateTime::from_unix_timestamp(2_000_000_000).expect("valid timestamp");
|
||||
extensions.insert(RequestedObjectLockRetainUntil(retain_until));
|
||||
let head = || RequestHead {
|
||||
method: Method::PUT,
|
||||
uri: "/bucket/key".parse().expect("valid URI"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
};
|
||||
|
||||
let envelope = RequestEnvelope::from_parts(head(), extensions);
|
||||
let empty = RequestEnvelope::from_parts(head(), Extensions::new());
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
envelope.multipart_staging_operation().is_some(),
|
||||
envelope.staged_multipart_part_authorization().is_some(),
|
||||
envelope
|
||||
.requested_object_lock_retention()
|
||||
.and_then(|value| value.0.mode.as_ref().map(|mode| mode.as_str())),
|
||||
envelope.requested_object_lock_retain_until().map(|value| value.0),
|
||||
envelope.residual_extensions().is_empty(),
|
||||
empty.multipart_staging_operation().is_none(),
|
||||
empty.staged_multipart_part_authorization().is_none(),
|
||||
empty.requested_object_lock_retention().is_none(),
|
||||
empty.requested_object_lock_retain_until().is_none(),
|
||||
),
|
||||
(true, true, Some("COMPLIANCE"), Some(retain_until), true, true, true, true, true),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_body_policy_payloads_move_from_extensions_into_named_fields() {
|
||||
let mut form_headers = HeaderMap::new();
|
||||
form_headers.insert("x-amz-tagging", http::HeaderValue::from_static("from=form"));
|
||||
let mut legal_hold = RequestedObjectLegalHold(Default::default());
|
||||
legal_hold.0.status = Some("ON".to_owned().into());
|
||||
let mut tags = RequestedObjectTags(Default::default());
|
||||
tags.0.tag_set.push(Default::default());
|
||||
tags.0.tag_set[0].key = Some("body".to_owned());
|
||||
let mut extensions = Extensions::new();
|
||||
extensions.insert(PostObjectPolicyHeaders(form_headers));
|
||||
extensions.insert(legal_hold);
|
||||
extensions.insert(tags);
|
||||
extensions.insert(RequestedCopySource("source-bucket/source-key".to_owned()));
|
||||
let head = || RequestHead {
|
||||
method: Method::PUT,
|
||||
uri: "/bucket/key".parse().expect("valid URI"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
};
|
||||
|
||||
let envelope = RequestEnvelope::from_parts(head(), extensions);
|
||||
let empty = RequestEnvelope::from_parts(head(), Extensions::new());
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
envelope
|
||||
.post_object_policy_headers()
|
||||
.and_then(|headers| headers.0.get("x-amz-tagging"))
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
envelope
|
||||
.requested_object_legal_hold()
|
||||
.and_then(|hold| hold.0.status.as_ref().map(|status| status.as_str())),
|
||||
envelope
|
||||
.requested_object_tags()
|
||||
.and_then(|tags| tags.0.tag_set.first())
|
||||
.and_then(|tag| tag.key.as_deref()),
|
||||
envelope.requested_copy_source().map(|source| source.0.as_str()),
|
||||
envelope.residual_extensions().is_empty(),
|
||||
empty.post_object_policy_headers().is_none(),
|
||||
empty.requested_object_legal_hold().is_none(),
|
||||
empty.requested_object_tags().is_none(),
|
||||
empty.requested_copy_source().is_none(),
|
||||
),
|
||||
(
|
||||
Some("from=form"),
|
||||
Some("ON"),
|
||||
Some("body"),
|
||||
Some("source-bucket/source-key"),
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_handed_over_secret_is_refused() {
|
||||
assert!(RequestCredentials::new("AKIAENVELOPE".to_owned(), String::new(), None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_absent_secret_matches_no_stored_secret() {
|
||||
let kept = RequestCredentials::without_secret_key("AKIAENVELOPE".to_owned(), None);
|
||||
assert_eq!(kept.secret_key(), None);
|
||||
assert!(!kept.secret_key_matches(""), "absent must not equal an empty stored secret");
|
||||
assert!(!kept.secret_key_matches("stored-secret"));
|
||||
let handed = RequestCredentials::new("AKIAENVELOPE".to_owned(), "stored-secret".to_owned(), None).expect("secret");
|
||||
assert!(!handed.secret_key_matches(""));
|
||||
assert!(!handed.secret_key_matches("other-secret"));
|
||||
assert!(handed.secret_key_matches("stored-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_reader_of_an_absent_secret_fails_closed() {
|
||||
let kept = RequestCredentials::without_secret_key("AKIAENVELOPE".to_owned(), Some("token".to_owned()));
|
||||
assert!(kept.require_secret_key().is_err());
|
||||
let handed = RequestCredentials::new("AKIAENVELOPE".to_owned(), "s".to_owned(), None).expect("secret");
|
||||
assert_eq!(handed.require_secret_key().ok(), Some("s"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kept_secret_debug_prints_neither_secret_nor_token() {
|
||||
let kept = RequestCredentials::without_secret_key("AKIAENVELOPE".to_owned(), Some("envelope-session-token".to_owned()));
|
||||
let debug = format!("{kept:?}");
|
||||
assert!(debug.contains("AKIAENVELOPE"), "{debug}");
|
||||
assert!(!debug.contains("envelope-session-token"), "{debug}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_credentials_debug_redacts_secret_and_session_token() {
|
||||
let credentials = RequestCredentials::new(
|
||||
"AKIAENVELOPE".to_owned(),
|
||||
"envelope-secret-value".to_owned(),
|
||||
Some("envelope-session-token".to_owned()),
|
||||
)
|
||||
.expect("non-empty secret");
|
||||
);
|
||||
|
||||
let debug = format!("{credentials:?}");
|
||||
|
||||
|
||||
@@ -36,7 +36,6 @@ pub(crate) mod trailer_adapter;
|
||||
// from here rather than through `app::context`.
|
||||
pub(crate) use context::RequestHead;
|
||||
pub use context::{RequestCredentials, RequestEnvelope};
|
||||
pub(crate) use context::{authorize_owned_internal_object_request, authorize_owned_request};
|
||||
|
||||
#[cfg(test)]
|
||||
mod capacity_dirty_scope_test;
|
||||
@@ -49,6 +48,4 @@ pub(crate) mod gating_test_env;
|
||||
#[cfg(test)]
|
||||
mod lifecycle_transition_api_test;
|
||||
#[cfg(test)]
|
||||
mod owned_authorization_test;
|
||||
#[cfg(test)]
|
||||
mod put_prelookup_gating_test;
|
||||
|
||||
+21
-4102
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,88 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! AbortMultipartUpload.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl DefaultMultipartUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_abort_multipart_upload(
|
||||
&self,
|
||||
req: S3Request<AbortMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<AbortMultipartUploadOutput>> {
|
||||
reject_presigned_multipart_max_total_object_size_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedSigV4Request>().is_some(),
|
||||
)?;
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
|
||||
)?;
|
||||
record_s3_op(S3Operation::AbortMultipartUpload);
|
||||
let mut opts = ObjectOptions::default();
|
||||
apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?;
|
||||
let AbortMultipartUploadInput {
|
||||
bucket, key, upload_id, ..
|
||||
} = req.input;
|
||||
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
// Per the AWS S3 API reference, a malformed upload ID is reported as
|
||||
// NoSuchUpload (404), the same as an upload that was aborted or
|
||||
// completed. See multipart_upload_api_error.
|
||||
match store
|
||||
.abort_multipart_upload(bucket.as_str(), key.as_str(), upload_id.as_str(), &opts)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
rustfs_scanner::record_dirty_usage_object_from_producer(
|
||||
&bucket,
|
||||
&key,
|
||||
rustfs_scanner::SegmentInvalidationProducerIdentity::AbortMultipartUpload,
|
||||
);
|
||||
Ok(S3Response::new(AbortMultipartUploadOutput { ..Default::default() }))
|
||||
}
|
||||
Err(err) => Err(multipart_upload_api_error(err)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized() {
|
||||
let input = AbortMultipartUploadInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::DELETE);
|
||||
let before = s3_op_total(S3Operation::AbortMultipartUpload);
|
||||
|
||||
let err = make_usecase().execute_abort_multipart_upload(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(s3_op_total(S3Operation::AbortMultipartUpload), before + 1);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,418 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! CreateMultipartUpload.
|
||||
|
||||
use super::*;
|
||||
|
||||
fn create_multipart_upload_metadata(
|
||||
input_metadata: Option<HashMap<String, String>>,
|
||||
headers: &HeaderMap,
|
||||
tagging: Option<String>,
|
||||
storage_class: Option<&crate::app::storage_api::s3::dto::StorageClass>,
|
||||
) -> HashMap<String, String> {
|
||||
let mut metadata = input_metadata.unwrap_or_default();
|
||||
namespace_reserved_user_metadata(&mut metadata);
|
||||
extract_metadata_from_mime(headers, &mut metadata);
|
||||
|
||||
if let Some(tags) = tagging {
|
||||
metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags);
|
||||
}
|
||||
if let Some(storage_class) = storage_class {
|
||||
metadata.insert(AMZ_STORAGE_CLASS.to_owned(), storage_class.as_str().to_owned());
|
||||
}
|
||||
|
||||
metadata
|
||||
}
|
||||
|
||||
/// A multipart session advertises disk compression only when the staged-rollout
|
||||
/// switch (`RUSTFS_COMPRESSION_MULTIPART_ENABLED`) is on, the object key/headers
|
||||
/// qualify, AND the session is not an SSE-C ciphertext-passthrough replication
|
||||
/// session, which must preserve source bytes verbatim.
|
||||
///
|
||||
/// The rollout switch defaults to off so a rolling upgrade never creates new
|
||||
/// compressed multipart objects while pre-fix nodes (whose decompressor is not
|
||||
/// resumable) may still serve reads. Enable it once the fleet has converged on a
|
||||
/// fixed build; the default flips per the `multipart-compression-default-off-window`
|
||||
/// entry in docs/architecture/compat-cleanup-register.md.
|
||||
///
|
||||
/// Each part is compressed as an independent stream; the GET path decodes across part
|
||||
/// boundaries (see `ReadTransform::Compressed`), so the session may advertise
|
||||
/// object-level compression again.
|
||||
///
|
||||
/// Unlike single PUT there is no `MIN_DISK_COMPRESSIBLE_SIZE` floor here: the total
|
||||
/// object size is unknown at CreateMultipartUpload time, so tiny multipart objects pay
|
||||
/// the (harmless) framing overhead. This is a deliberate trade-off, not a bug.
|
||||
fn should_advertise_session_compression(multipart_enabled: bool, ciphertext_passthrough: bool, disk_compressible: bool) -> bool {
|
||||
multipart_enabled && !ciphertext_passthrough && disk_compressible
|
||||
}
|
||||
|
||||
impl DefaultMultipartUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
#[hotpath::measure(impl_type = "MultipartUsecase")]
|
||||
pub async fn execute_create_multipart_upload(
|
||||
&self,
|
||||
req: S3Request<CreateMultipartUploadInput>,
|
||||
) -> S3Result<S3Response<CreateMultipartUploadOutput>> {
|
||||
validate_new_object_key(&req.input.key, "CreateMultipartUpload")?;
|
||||
let multipart_max_total_object_size = parse_presigned_multipart_max_total_object_size(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedSigV4Request>().is_some(),
|
||||
)?;
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
|
||||
)?;
|
||||
let helper =
|
||||
OperationHelper::new(&req, EventName::ObjectCreatedCreateMultipartUpload, S3Operation::CreateMultipartUpload)
|
||||
.suppress_event();
|
||||
let replication_authorized = replication_request_authorized(&req);
|
||||
let CreateMultipartUploadInput {
|
||||
bucket,
|
||||
key,
|
||||
tagging,
|
||||
version_id,
|
||||
storage_class,
|
||||
server_side_encryption,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key,
|
||||
sse_customer_key_md5,
|
||||
ssekms_key_id,
|
||||
object_lock_legal_hold_status,
|
||||
object_lock_mode,
|
||||
object_lock_retain_until_date,
|
||||
metadata: input_metadata,
|
||||
..
|
||||
} = req.input.clone();
|
||||
|
||||
let server_side_encryption = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?);
|
||||
let ssekms_key_id = ssekms_key_id.or_else(|| {
|
||||
req.headers
|
||||
.get("x-amz-server-side-encryption-aws-kms-key-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(ToOwned::to_owned)
|
||||
});
|
||||
|
||||
// Validate storage class if provided
|
||||
if let Some(ref storage_class) = storage_class
|
||||
&& !is_valid_storage_class(storage_class.as_str())
|
||||
{
|
||||
return Err(s3_error!(InvalidStorageClass));
|
||||
}
|
||||
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
validate_archive_content_encoding(
|
||||
&key,
|
||||
req.headers.get("content-type").and_then(|value| value.to_str().ok()),
|
||||
req.headers.get("content-encoding").and_then(|value| value.to_str().ok()),
|
||||
)?;
|
||||
|
||||
let mut metadata = create_multipart_upload_metadata(input_metadata, &req.headers, tagging, storage_class.as_ref());
|
||||
if let Some(limit) = multipart_max_total_object_size {
|
||||
insert_str(&mut metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE, limit.to_string());
|
||||
}
|
||||
|
||||
let has_explicit_object_lock_retention = object_lock_mode.is_some()
|
||||
|| object_lock_retain_until_date.is_some()
|
||||
|| has_replication_retention_update(&req.headers, replication_authorized);
|
||||
let object_lock_config_state = load_bucket_object_lock_config_state(&bucket).await?;
|
||||
if let Some(object_lock_metadata) = build_put_like_object_lock_metadata(
|
||||
&bucket,
|
||||
&object_lock_config_state,
|
||||
object_lock_legal_hold_status,
|
||||
object_lock_mode,
|
||||
object_lock_retain_until_date,
|
||||
)? {
|
||||
metadata.extend(object_lock_metadata);
|
||||
}
|
||||
apply_bucket_default_lock_retention(
|
||||
&bucket,
|
||||
&object_lock_config_state,
|
||||
&mut metadata,
|
||||
has_explicit_object_lock_retention,
|
||||
)?;
|
||||
let (header_sse_customer_algorithm, header_sse_customer_key, header_sse_customer_key_md5) =
|
||||
extract_ssec_params_from_headers(&req.headers)?;
|
||||
let sse_customer_algorithm = sse_customer_algorithm.or(header_sse_customer_algorithm);
|
||||
let sse_customer_key = sse_customer_key.or(header_sse_customer_key);
|
||||
let sse_customer_key_md5 = sse_customer_key_md5.or(header_sse_customer_key_md5);
|
||||
|
||||
// The session data key is generated here, so this is where a multipart upload is held
|
||||
// to the KMS key it names. Parts and completion also authorize their own caller.
|
||||
let session_principal = SseKmsPrincipal::from_request(&req);
|
||||
let encryption_request = PrepareEncryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
server_side_encryption,
|
||||
ssekms_key_id,
|
||||
ssekms_context: extract_ssekms_context_from_headers(&req.headers)?,
|
||||
sse_customer_algorithm: sse_customer_algorithm.clone(),
|
||||
sse_customer_key,
|
||||
sse_customer_key_md5: sse_customer_key_md5.clone(),
|
||||
principal: session_principal.as_ref(),
|
||||
};
|
||||
|
||||
// SSE-C ciphertext passthrough: parts are already encrypted, so no
|
||||
// session DEK is prepared; a session marker tells UploadPart to store
|
||||
// the ciphertext verbatim instead of recovering encryption material.
|
||||
let ciphertext_passthrough = replication_authorized
|
||||
&& get_header(&req.headers, SUFFIX_SOURCE_REPLICATION_REQUEST).as_deref() == Some("true")
|
||||
&& rustfs_utils::http::ssec_transport_to_stored_metadata(&req.headers).is_some();
|
||||
if ciphertext_passthrough && let Some(metadata_sys) = self.bucket_metadata_sys() {
|
||||
let check_result = map_quota_check_outcome(
|
||||
&bucket,
|
||||
QuotaChecker::new(metadata_sys)
|
||||
.check_quota(&bucket, QuotaOperation::PutObject, 0)
|
||||
.await,
|
||||
)?;
|
||||
if check_result.quota_limit.is_some() {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
"SSE-C ciphertext replication is unavailable for quota-enabled buckets".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
if ciphertext_passthrough {
|
||||
insert_str(&mut metadata, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT, "true".to_string());
|
||||
}
|
||||
|
||||
let prepared_material = if ciphertext_passthrough {
|
||||
None
|
||||
} else {
|
||||
sse_prepare_encryption(encryption_request).await?
|
||||
};
|
||||
let (effective_sse, effective_kms_key_id) = match prepared_material {
|
||||
Some(material) => {
|
||||
let server_side_encryption = Some(material.server_side_encryption.clone());
|
||||
let ssekms_key_id = material.response_kms_key_id();
|
||||
|
||||
let mut encryption_metadata = encryption_material_to_metadata(&material)?;
|
||||
if material.key_kind == EncryptionKeyKind::Object {
|
||||
mark_encrypted_multipart_metadata(&mut encryption_metadata);
|
||||
}
|
||||
metadata.extend(encryption_metadata);
|
||||
|
||||
(server_side_encryption, ssekms_key_id)
|
||||
}
|
||||
None => (None, None),
|
||||
};
|
||||
|
||||
if should_advertise_session_compression(
|
||||
is_multipart_disk_compression_enabled(),
|
||||
ciphertext_passthrough,
|
||||
is_disk_compressible(&req.headers, &key),
|
||||
) {
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
compression_metadata_value(CompressionAlgorithm::default()),
|
||||
);
|
||||
}
|
||||
|
||||
let mt2 = metadata.clone();
|
||||
let mut opts: ObjectOptions =
|
||||
put_opts_with_replication_authorization(&bucket, &key, version_id, &req.headers, metadata, replication_authorized)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
apply_bucket_generation_guard(&req, &bucket, &mut opts)?;
|
||||
|
||||
let dsc =
|
||||
must_replicate_object(&bucket, &key, &mt2, "".to_string(), opts.delete_marker_replication_status(), opts.clone())
|
||||
.await;
|
||||
if dsc.replicate_any() {
|
||||
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_GENERATION, Uuid::new_v4().to_string());
|
||||
insert_str(&mut opts.user_defined, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
|
||||
insert_str(
|
||||
&mut opts.user_defined,
|
||||
SUFFIX_REPLICATION_STATUS,
|
||||
dsc.pending_status().unwrap_or_default(),
|
||||
);
|
||||
}
|
||||
|
||||
if !super::super::object::shared::put_like_write_creates_new_version(&opts)
|
||||
&& super::super::object::shared::object_lock_checks_required_for_state(&object_lock_config_state)
|
||||
{
|
||||
let current_opts: ObjectOptions = internal_object_info_lookup_opts(
|
||||
get_opts(&bucket, &key, opts.version_id.clone(), None, &req.headers)
|
||||
.await
|
||||
.map_err(ApiError::from)?,
|
||||
);
|
||||
match store.get_object_info(&bucket, &key, ¤t_opts).await {
|
||||
Ok(existing_obj_info) => {
|
||||
validate_existing_object_lock_for_write(&object_lock_config_state, &existing_obj_info, &opts)?
|
||||
}
|
||||
Err(err) => {
|
||||
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let checksum_type = rustfs_rio::ChecksumType::from_header(&req.headers);
|
||||
if checksum_type.is(rustfs_rio::ChecksumType::INVALID) {
|
||||
return Err(s3_error!(InvalidArgument, "Invalid checksum type"));
|
||||
} else if checksum_type.is_set() && !checksum_type.is(rustfs_rio::ChecksumType::TRAILING) {
|
||||
opts.want_checksum = Some(rustfs_rio::Checksum {
|
||||
checksum_type,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
if effective_sse.is_some() && opts.want_checksum.is_some() {
|
||||
insert_str(&mut opts.user_defined, SUFFIX_PLAINTEXT_CHECKSUM, "true".to_string());
|
||||
}
|
||||
|
||||
let MultipartUploadResult {
|
||||
upload_id,
|
||||
checksum_algo,
|
||||
checksum_type,
|
||||
} = store
|
||||
.new_multipart_upload(&bucket, &key, &opts)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let output = CreateMultipartUploadOutput {
|
||||
bucket: Some(bucket),
|
||||
key: Some(key),
|
||||
upload_id: Some(upload_id),
|
||||
server_side_encryption: effective_sse,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key_md5,
|
||||
ssekms_key_id: effective_kms_key_id,
|
||||
checksum_algorithm: checksum_algo.map(ChecksumAlgorithm::from),
|
||||
checksum_type: checksum_type.map(ChecksumType::from),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = Ok(S3Response::new(output));
|
||||
let _ = helper.complete(&result);
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn session_compression_is_advertised_only_for_non_passthrough_compressible_uploads() {
|
||||
// (multipart_enabled, ciphertext_passthrough, disk_compressible, expected)
|
||||
let cases = [
|
||||
(true, false, false, false),
|
||||
(true, false, true, true),
|
||||
(true, true, false, false),
|
||||
(true, true, true, false),
|
||||
// The staged-rollout switch keeps multipart compression dark by
|
||||
// default regardless of the other gates.
|
||||
(false, false, true, false),
|
||||
(false, false, false, false),
|
||||
(false, true, true, false),
|
||||
(false, true, false, false),
|
||||
];
|
||||
|
||||
for (multipart_enabled, ciphertext_passthrough, disk_compressible, expected) in cases {
|
||||
assert_eq!(
|
||||
should_advertise_session_compression(multipart_enabled, ciphertext_passthrough, disk_compressible),
|
||||
expected,
|
||||
"multipart_enabled={multipart_enabled} ciphertext_passthrough={ciphertext_passthrough} disk_compressible={disk_compressible}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_create_multipart_upload_rejects_invalid_storage_class() {
|
||||
let input = CreateMultipartUploadInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.storage_class(Some(StorageClass::from("invalid".to_string())))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::POST);
|
||||
|
||||
let err = make_usecase().execute_create_multipart_upload(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidStorageClass);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_multipart_upload_metadata_persists_dto_storage_class_without_raw_header() {
|
||||
let metadata = create_multipart_upload_metadata(
|
||||
None,
|
||||
&HeaderMap::new(),
|
||||
None,
|
||||
Some(&StorageClass::from_static("REDUCED_REDUNDANCY")),
|
||||
);
|
||||
|
||||
assert_eq!(metadata.get(AMZ_STORAGE_CLASS), Some(&"REDUCED_REDUNDANCY".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_multipart_upload_metadata_keeps_user_and_system_namespaces_separate() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("content-type", HeaderValue::from_static("application/octet-stream"));
|
||||
headers.insert(AMZ_STORAGE_CLASS, HeaderValue::from_static("STANDARD"));
|
||||
let input_metadata = HashMap::from([
|
||||
("content-type".to_string(), "user-content-type".to_string()),
|
||||
(AMZ_STORAGE_CLASS.to_string(), "user-storage-class".to_string()),
|
||||
]);
|
||||
|
||||
let metadata = create_multipart_upload_metadata(
|
||||
Some(input_metadata),
|
||||
&headers,
|
||||
Some("project=rustfs".to_string()),
|
||||
Some(&StorageClass::from_static("REDUCED_REDUNDANCY")),
|
||||
);
|
||||
|
||||
assert_eq!(metadata.get("content-type"), Some(&"application/octet-stream".to_string()));
|
||||
assert_eq!(metadata.get("x-amz-meta-content-type"), Some(&"user-content-type".to_string()));
|
||||
assert_eq!(metadata.get(AMZ_STORAGE_CLASS), Some(&"REDUCED_REDUNDANCY".to_string()));
|
||||
assert_eq!(metadata.get("x-amz-meta-x-amz-storage-class"), Some(&"user-storage-class".to_string()));
|
||||
assert_eq!(metadata.get(AMZ_OBJECT_TAGGING), Some(&"project=rustfs".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn create_multipart_rejects_ciphertext_replication_before_parts_are_staged() {
|
||||
let (_store, bucket) = crate::app::gating_test_env::durable_quota_test_bucket("ciphertext-multipart-quota", 4096).await;
|
||||
let usecase = DefaultMultipartUsecase::from_global();
|
||||
let input = CreateMultipartUploadInput::builder()
|
||||
.bucket(bucket)
|
||||
.key("object".to_string())
|
||||
.build()
|
||||
.expect("create multipart request should build");
|
||||
let mut request = build_request(input, Method::POST);
|
||||
insert_header(&mut request.headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
request
|
||||
.headers
|
||||
.insert(rustfs_utils::http::REPLICATION_SSEC_ALGORITHM_HEADER, HeaderValue::from_static("AES256"));
|
||||
request.extensions.insert(crate::storage::access::ReqInfo {
|
||||
replication_request_authorized: true,
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = usecase
|
||||
.execute_create_multipart_upload(request)
|
||||
.await
|
||||
.expect_err("quota-enabled ciphertext multipart replication should fail before upload creation");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! ListMultipartUploads and ListParts.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl DefaultMultipartUsecase {
|
||||
pub async fn execute_list_multipart_uploads(
|
||||
&self,
|
||||
req: S3Request<ListMultipartUploadsInput>,
|
||||
) -> S3Result<S3Response<ListMultipartUploadsOutput>> {
|
||||
let table_list_access = req.extensions.get::<TableDataPlaneListAccess>().cloned();
|
||||
reject_presigned_multipart_max_total_object_size_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedSigV4Request>().is_some(),
|
||||
)?;
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
|
||||
)?;
|
||||
let mut opts = ObjectOptions::default();
|
||||
apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?;
|
||||
let ListMultipartUploadsInput {
|
||||
bucket,
|
||||
prefix,
|
||||
delimiter,
|
||||
key_marker,
|
||||
upload_id_marker,
|
||||
max_uploads,
|
||||
encoding_type,
|
||||
..
|
||||
} = req.input;
|
||||
|
||||
let ListMultipartUploadsParams {
|
||||
prefix,
|
||||
key_marker,
|
||||
max_uploads,
|
||||
} = parse_list_multipart_uploads_params(prefix, key_marker, max_uploads)?;
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
// `apply_bucket_generation_guard` tolerates a missing guard (only the S3
|
||||
// access layer installs one), so resolve the current generation rather
|
||||
// than failing the request. Listing is filtered by this value, so a
|
||||
// stale one simply hides foreign-incarnation uploads, as intended.
|
||||
let expected_incarnation_id = match opts.expected_bucket_incarnation_id {
|
||||
Some(incarnation_id) => incarnation_id,
|
||||
None => store.bucket_incarnation_id_from_disk(&bucket).await.map_err(ApiError::from)?,
|
||||
};
|
||||
|
||||
let result = match table_list_access.as_ref() {
|
||||
Some(access) => {
|
||||
table_list_isolation::list_multipart_uploads(
|
||||
store.clone(),
|
||||
access,
|
||||
table_list_isolation::ListMultipartUploadsRequest {
|
||||
bucket: &bucket,
|
||||
prefix: &prefix,
|
||||
key_marker: key_marker.as_deref(),
|
||||
upload_id_marker: upload_id_marker.as_deref(),
|
||||
delimiter: delimiter.as_deref(),
|
||||
max_uploads,
|
||||
expected_incarnation_id,
|
||||
},
|
||||
)
|
||||
.await?
|
||||
}
|
||||
None => store
|
||||
.list_multipart_uploads_for_bucket_incarnation(
|
||||
&bucket,
|
||||
&prefix,
|
||||
key_marker,
|
||||
upload_id_marker,
|
||||
delimiter,
|
||||
max_uploads,
|
||||
expected_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?,
|
||||
};
|
||||
|
||||
Ok(S3Response::new(build_list_multipart_uploads_output(
|
||||
bucket,
|
||||
prefix,
|
||||
result,
|
||||
encoding_type.as_ref(),
|
||||
)))
|
||||
}
|
||||
|
||||
pub async fn execute_list_parts(&self, req: S3Request<ListPartsInput>) -> S3Result<S3Response<ListPartsOutput>> {
|
||||
reject_presigned_multipart_max_total_object_size_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedSigV4Request>().is_some(),
|
||||
)?;
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
|
||||
)?;
|
||||
let mut opts = ObjectOptions::default();
|
||||
apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?;
|
||||
let ListPartsInput {
|
||||
bucket,
|
||||
key,
|
||||
upload_id,
|
||||
part_number_marker,
|
||||
max_parts,
|
||||
..
|
||||
} = req.input;
|
||||
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
|
||||
let params = parse_list_parts_params(part_number_marker, max_parts)?;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let res = store
|
||||
.list_object_parts(&bucket, &key, &upload_id, params.part_number_marker, params.max_parts, &opts)
|
||||
.await
|
||||
.map_err(multipart_upload_api_error)?;
|
||||
|
||||
Ok(S3Response::new(build_list_parts_output(res)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_multipart_uploads_returns_internal_error_when_store_uninitialized() {
|
||||
let input = ListMultipartUploadsInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::GET);
|
||||
|
||||
let err = make_usecase().execute_list_multipart_uploads(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_multipart_uploads_rejects_invalid_key_marker_before_store_lookup() {
|
||||
let input = ListMultipartUploadsInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.prefix(Some("prefix/".to_string()))
|
||||
.key_marker(Some("other/key".to_string()))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::GET);
|
||||
|
||||
let err = make_usecase().execute_list_multipart_uploads(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("Invalid key marker"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_multipart_uploads_rejects_invalid_max_uploads_before_store_lookup() {
|
||||
let input = ListMultipartUploadsInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.max_uploads(Some(0))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::GET);
|
||||
let expected = "max-uploads must be between 1 and 1000";
|
||||
|
||||
let err = make_usecase().execute_list_multipart_uploads(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some(expected));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_parts_returns_internal_error_when_store_uninitialized() {
|
||||
let input = ListPartsInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::GET);
|
||||
|
||||
let err = make_usecase().execute_list_parts(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_parts_rejects_negative_part_number_marker_before_store_lookup() {
|
||||
let input = ListPartsInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.part_number_marker(Some(-1))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::GET);
|
||||
|
||||
let err = make_usecase().execute_list_parts(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("part-number-marker must be non-negative"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_parts_rejects_invalid_max_parts_before_store_lookup() {
|
||||
let input = ListPartsInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.max_parts(Some(1001))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::GET);
|
||||
|
||||
let err = make_usecase().execute_list_parts(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("max-parts must be between 1 and 1000"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Fixtures and test-only imports shared by the multipart use-case tests.
|
||||
|
||||
use super::*;
|
||||
pub(super) use crate::app::storage_api::s3::dto::CompletedMultipartUpload;
|
||||
pub(super) use crate::app::storage_api::s3::dto::StorageClass;
|
||||
pub(super) use http::Extensions;
|
||||
pub(super) use http::Method;
|
||||
pub(super) use rustfs_filemeta::ObjectPartInfo;
|
||||
pub(super) use rustfs_utils::http::AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER;
|
||||
pub(super) use rustfs_utils::http::AMZ_OBJECT_LOCK_MODE_LOWER;
|
||||
pub(super) use rustfs_utils::http::AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER;
|
||||
pub(super) use std::io::Cursor;
|
||||
pub(super) use temp_env::async_with_vars;
|
||||
pub(super) use tokio::io::AsyncReadExt;
|
||||
|
||||
pub(super) fn s3_op_total(op: S3Operation) -> u64 {
|
||||
rustfs_io_metrics::s3_op_metrics_snapshot()
|
||||
.into_iter()
|
||||
.find(|snapshot| snapshot.op == op.as_str())
|
||||
.map(|snapshot| snapshot.total)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(super) fn build_request<T>(input: T, method: Method) -> S3Request<T> {
|
||||
S3Request {
|
||||
input,
|
||||
method,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn make_usecase() -> DefaultMultipartUsecase {
|
||||
DefaultMultipartUsecase::without_context()
|
||||
}
|
||||
@@ -0,0 +1,823 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! UploadPart.
|
||||
|
||||
use super::*;
|
||||
|
||||
fn resolve_upload_part_size(content_length: Option<i64>, body: Option<&StreamingBlob>) -> rustfs_s3_types::S3Result<Option<i64>> {
|
||||
if let Some(length) = content_length {
|
||||
return Ok(Some(length));
|
||||
}
|
||||
body.and_then(|body| body.remaining_length().exact())
|
||||
.map(i64::try_from)
|
||||
.transpose()
|
||||
.map_err(|_| rustfs_s3_types::s3_error!(UnexpectedContent))
|
||||
}
|
||||
|
||||
fn require_upload_part_size(size: Option<i64>, capped: bool) -> rustfs_s3_types::S3Result<i64> {
|
||||
match size {
|
||||
Some(size) if size >= 0 => Ok(size),
|
||||
Some(_) => Err(rustfs_s3_types::s3_error!(UnexpectedContent)),
|
||||
None if capped => Err(rustfs_s3_types::s3_error!(UnexpectedContent)),
|
||||
None => Err(rustfs_s3_types::S3Error::new(rustfs_s3_types::S3ErrorCode::MissingContentLength)),
|
||||
}
|
||||
}
|
||||
|
||||
fn upload_part_body_read_timeout(configured: Duration, capped: bool) -> Duration {
|
||||
if capped {
|
||||
configured.max(Duration::from_secs(rustfs_config::DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT))
|
||||
} else {
|
||||
configured
|
||||
}
|
||||
}
|
||||
|
||||
/// Plaintext length of one SSE-C passthrough part, declared by the sender on
|
||||
/// UploadPart (backlog#2363).
|
||||
fn passthrough_part_actual_size(headers: &HeaderMap) -> Option<i64> {
|
||||
get_header(headers, rustfs_utils::http::SUFFIX_REPLICATION_PART_ACTUAL_SIZE)?
|
||||
.parse::<i64>()
|
||||
.ok()
|
||||
.filter(|size| *size > 0)
|
||||
}
|
||||
|
||||
// Hoisted values are signed query parameters, not HTTP headers. Normalize only
|
||||
// after access control has verified the presigned request.
|
||||
fn normalize_presigned_part_checksums(req: &mut S3Request<UploadPartInput>) -> S3Result<()> {
|
||||
if req.extensions.get::<VerifiedPresignedRequest>().is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(query) = req.uri.query() else {
|
||||
return Ok(());
|
||||
};
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for (name, value) in url::form_urlencoded::parse(query.as_bytes()) {
|
||||
let field = match name.as_ref() {
|
||||
"x-amz-checksum-crc32" => Some(&mut req.input.checksum_crc32),
|
||||
"x-amz-checksum-crc32c" => Some(&mut req.input.checksum_crc32c),
|
||||
"x-amz-checksum-crc64nvme" => Some(&mut req.input.checksum_crc64nvme),
|
||||
"x-amz-checksum-sha1" => Some(&mut req.input.checksum_sha1),
|
||||
"x-amz-checksum-sha256" => Some(&mut req.input.checksum_sha256),
|
||||
"x-amz-sdk-checksum-algorithm" => None,
|
||||
_ => continue,
|
||||
};
|
||||
if !seen.insert(name.clone()) {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InvalidRequest, "Duplicate checksum query parameter"));
|
||||
}
|
||||
if let Some(header) = req.headers.get(name.as_ref())
|
||||
&& header.as_bytes() != value.as_bytes()
|
||||
{
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
"Conflicting checksum header and query parameter",
|
||||
));
|
||||
}
|
||||
let header = http::HeaderValue::from_str(&value)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid checksum query parameter"))?;
|
||||
req.headers.insert(
|
||||
http::header::HeaderName::from_bytes(name.as_bytes())
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidArgument, "Invalid checksum query parameter"))?,
|
||||
header,
|
||||
);
|
||||
if let Some(field) = field {
|
||||
*field = Some(value.into_owned());
|
||||
} else {
|
||||
req.input.checksum_algorithm = Some(ChecksumAlgorithm::from(value.into_owned()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl DefaultMultipartUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
#[hotpath::measure(impl_type = "MultipartUsecase")]
|
||||
pub async fn execute_upload_part(&self, mut req: S3Request<UploadPartInput>) -> S3Result<S3Response<UploadPartOutput>> {
|
||||
reject_presigned_multipart_max_total_object_size_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedSigV4Request>().is_some(),
|
||||
)?;
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
|
||||
)?;
|
||||
normalize_presigned_part_checksums(&mut req)?;
|
||||
let mut opts = ObjectOptions::default();
|
||||
apply_bucket_generation_guard(&req, &req.input.bucket, &mut opts)?;
|
||||
let part_principal = SseKmsPrincipal::from_request(&req);
|
||||
let input = req.input;
|
||||
let UploadPartInput {
|
||||
body,
|
||||
bucket,
|
||||
key,
|
||||
upload_id,
|
||||
part_number,
|
||||
content_length,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key,
|
||||
sse_customer_key_md5,
|
||||
// content_md5,
|
||||
..
|
||||
} = input;
|
||||
|
||||
let part_id = parse_upload_part_number(part_number)?;
|
||||
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
|
||||
let size = resolve_upload_part_size(content_length, body.as_ref())?;
|
||||
if let Some(size) = size {
|
||||
reject_oversize_single_upload(size)?;
|
||||
}
|
||||
let body_stream = body.ok_or_else(|| s3_error!(IncompleteBody))?;
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
let fi = store
|
||||
.get_multipart_info(&bucket, &key, &upload_id, &opts)
|
||||
.await
|
||||
.map_err(multipart_upload_api_error)?;
|
||||
authorize_sse_kms_multipart_write(part_principal.as_ref(), &fi.user_defined).await?;
|
||||
let max_total_object_size = multipart_max_total_object_size(&fi.user_defined)?;
|
||||
let mut size = require_upload_part_size(size, max_total_object_size.is_some())?;
|
||||
if let Some(limit) = max_total_object_size
|
||||
&& u64::try_from(size).is_ok_and(|size| size > limit)
|
||||
{
|
||||
return Err(S3Error::new(S3ErrorCode::EntityTooLarge));
|
||||
}
|
||||
let admission_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
let admission = self.concurrency_manager().admit_multipart_part(size).await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("multipart_admission_wait", admission_wait_started);
|
||||
let upload_part_admission = match admission
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "foreground write admission closed"))?
|
||||
{
|
||||
ForegroundWriteAdmission::Disabled => None,
|
||||
ForegroundWriteAdmission::Admitted(permit) => {
|
||||
counter!("rustfs.upload_part.foreground_admission.total", "result" => "admitted").increment(1);
|
||||
Some(permit)
|
||||
}
|
||||
ForegroundWriteAdmission::Rejected => {
|
||||
counter!("rustfs.upload_part.foreground_admission.total", "result" => "rejected").increment(1);
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::SlowDown,
|
||||
"foreground write concurrency limit reached, please reduce your request rate",
|
||||
));
|
||||
}
|
||||
};
|
||||
let request_id = req
|
||||
.extensions
|
||||
.get::<super::super::storage_api::multipart_usecase::request_context::RequestContext>()
|
||||
.map(|ctx| ctx.request_id.as_str())
|
||||
.unwrap_or_default();
|
||||
let timeout = upload_part_body_read_timeout(put_object_body_read_timeout(), max_total_object_size.is_some());
|
||||
let (body_stream, progress_control) =
|
||||
if let Some(control) = req.extensions.get::<super::super::object::request_body::BodyReadControl>() {
|
||||
control.activate_upload(timeout, &bucket, &key, request_id, size.max(0) as u64);
|
||||
(body_stream, control.clone())
|
||||
} else {
|
||||
super::super::object::request_body::guard_upload_progress(
|
||||
body_stream,
|
||||
timeout,
|
||||
&bucket,
|
||||
&key,
|
||||
request_id,
|
||||
size.max(0) as u64,
|
||||
)
|
||||
};
|
||||
|
||||
let ingress_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
|
||||
// Apply adaptive buffer sizing based on part size for optimal streaming performance.
|
||||
// Uses workload profile configuration (enabled by default) to select appropriate buffer size.
|
||||
// Buffer sizes range from 32KB to 4MB depending on part size and configured workload profile.
|
||||
let buffer_size = get_buffer_size_opt_in(size);
|
||||
let body = tokio::io::BufReader::with_capacity(
|
||||
buffer_size,
|
||||
StreamReader::new(body_stream.map(|f| f.map_err(s3s_body_error_to_io))),
|
||||
);
|
||||
|
||||
// An SSE-C passthrough session stores ciphertext parts verbatim: a
|
||||
// compression key restored on the session describes those stored
|
||||
// bytes and must not add a second compression layer, and each part's
|
||||
// plaintext length comes from the sender (backlog#2363).
|
||||
let preserve_ciphertext = contains_key_str(&fi.user_defined, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT);
|
||||
let is_disk_compressed = !preserve_ciphertext
|
||||
&& rustfs_utils::http::contains_key_str(&fi.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
|
||||
|
||||
let actual_size = if preserve_ciphertext {
|
||||
passthrough_part_actual_size(&req.headers).unwrap_or(size)
|
||||
} else {
|
||||
size
|
||||
};
|
||||
|
||||
let mut md5hex = if let Some(base64_md5) = input.content_md5 {
|
||||
let md5 = base64_simd::STANDARD
|
||||
.decode_to_vec(base64_md5.as_bytes())
|
||||
.map_err(|e| ApiError::from(StorageError::other(format!("Invalid content MD5: {e}"))))?;
|
||||
Some(hex_simd::encode_to_string(&md5, hex_simd::AsciiCase::Lower))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut sha256hex = get_content_sha256_with_query(&req.headers, req.uri.query());
|
||||
|
||||
let mut write_plan = WritePlan::new();
|
||||
let mut reader = if is_disk_compressed {
|
||||
let algorithm = CompressionAlgorithm::default();
|
||||
let mut hrd = HashReader::from_stream(body, size, actual_size, md5hex.take(), sha256hex.take(), false)
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
if let Err(err) = hrd.add_checksum(&req.headers, trailer_source(req.trailing_headers.clone()), false) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
|
||||
size = HashReader::SIZE_PRESERVE_LAYER;
|
||||
write_plan = write_plan.with_compression(algorithm);
|
||||
hrd
|
||||
} else {
|
||||
HashReader::from_stream(body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
|
||||
};
|
||||
|
||||
if let Err(err) = reader.add_checksum(&req.headers, trailer_source(req.trailing_headers.clone()), size < 0) {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
opts.want_checksum = reader.checksum();
|
||||
|
||||
// An SSE-C passthrough session stores ciphertext parts verbatim: no
|
||||
// material recovery, no validation against the (absent) customer key.
|
||||
let has_ssec = !preserve_ciphertext
|
||||
&& fi
|
||||
.user_defined
|
||||
.contains_key("x-amz-server-side-encryption-customer-algorithm");
|
||||
let (server_side_encryption, ssekms_key_id) = if has_ssec || preserve_ciphertext {
|
||||
(None, None)
|
||||
} else {
|
||||
let sse = fi
|
||||
.user_defined
|
||||
.get("x-amz-server-side-encryption")
|
||||
.map(|s| {
|
||||
ServerSideEncryption::from_str(s)
|
||||
.map_err(|e| ApiError::from(StorageError::other(format!("Invalid server-side encryption: {e}"))))
|
||||
})
|
||||
.transpose()?;
|
||||
let key_id = match sse.as_ref() {
|
||||
Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => fi
|
||||
.user_defined
|
||||
.get("x-amz-server-side-encryption-aws-kms-key-id")
|
||||
.map(|s| s.to_string()),
|
||||
_ => None,
|
||||
};
|
||||
(sse, key_id)
|
||||
};
|
||||
if !preserve_ciphertext {
|
||||
EncryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
server_side_encryption: server_side_encryption.clone(),
|
||||
ssekms_key_id: ssekms_key_id.clone(),
|
||||
ssekms_context: None,
|
||||
sse_customer_algorithm: sse_customer_algorithm.clone(),
|
||||
sse_customer_key: sse_customer_key.clone(),
|
||||
sse_customer_key_md5: sse_customer_key_md5.clone(),
|
||||
content_size: actual_size,
|
||||
principal: None,
|
||||
}
|
||||
.validate_multipart_ssec(&fi.user_defined)?;
|
||||
}
|
||||
let (requested_sse, requested_kms_key_id) = if has_ssec {
|
||||
let ssec_material = sse_decryption(DecryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
metadata: &fi.user_defined,
|
||||
sse_customer_key: sse_customer_key.as_ref(),
|
||||
sse_customer_key_md5: sse_customer_key_md5.as_ref(),
|
||||
principal: None,
|
||||
})
|
||||
.await?
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
|
||||
let ssec_write = match ssec_material.key_kind {
|
||||
EncryptionKeyKind::Object => WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32),
|
||||
EncryptionKeyKind::Direct => {
|
||||
WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id)
|
||||
}
|
||||
};
|
||||
write_plan = write_plan.with_encryption(ssec_write);
|
||||
(Some(ssec_material.server_side_encryption), ssec_material.kms_key_id)
|
||||
} else if let Some(server_side_encryption) = server_side_encryption {
|
||||
// Reuse the stored envelope after authorizing this caller against its
|
||||
// pinned destination key above.
|
||||
let managed_material = sse_decryption(DecryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
metadata: &fi.user_defined,
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
principal: None,
|
||||
})
|
||||
.await?
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
|
||||
let managed_write = match managed_material.key_kind {
|
||||
EncryptionKeyKind::Object => WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32),
|
||||
EncryptionKeyKind::Direct => {
|
||||
WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_id)
|
||||
}
|
||||
};
|
||||
write_plan = write_plan.with_encryption(managed_write);
|
||||
(Some(server_side_encryption), ssekms_key_id)
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
|
||||
|
||||
{
|
||||
use rustfs_rio::HashReaderMut;
|
||||
let inner = reader.take_inner();
|
||||
reader.inner =
|
||||
rustfs_rio::boxed_reader(super::super::object::request_body::DemandReader::new(inner, progress_control));
|
||||
}
|
||||
|
||||
let mut reader = PutObjReader::new(reader);
|
||||
|
||||
if let Some(stage_start) = ingress_stage_start {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
"multipart_ingress_prepare",
|
||||
stage_start.elapsed().as_secs_f64() * 1000.0,
|
||||
);
|
||||
}
|
||||
|
||||
let _upload_part_admission = upload_part_admission;
|
||||
let store_write_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = store
|
||||
.put_object_part(&bucket, &key, &upload_id, part_id, &mut reader, &opts)
|
||||
.await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("multipart_store_write", store_write_started);
|
||||
drop(_upload_part_admission);
|
||||
let info = result.map_err(ApiError::from)?;
|
||||
|
||||
let mut checksum_crc32 = input.checksum_crc32;
|
||||
let mut checksum_crc32c = input.checksum_crc32c;
|
||||
let mut checksum_sha1 = input.checksum_sha1;
|
||||
let mut checksum_sha256 = input.checksum_sha256;
|
||||
let mut checksum_crc64nvme = input.checksum_crc64nvme;
|
||||
|
||||
if let Some(alg) = &input.checksum_algorithm
|
||||
&& let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| {
|
||||
let key = match alg.as_str() {
|
||||
ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(),
|
||||
ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(),
|
||||
ChecksumAlgorithm::SHA1 => rustfs_rio::ChecksumType::SHA1.key(),
|
||||
ChecksumAlgorithm::SHA256 => rustfs_rio::ChecksumType::SHA256.key(),
|
||||
ChecksumAlgorithm::CRC64NVME => rustfs_rio::ChecksumType::CRC64_NVME.key(),
|
||||
_ => return None,
|
||||
};
|
||||
trailer.read(|headers| {
|
||||
headers
|
||||
.get(key.unwrap_or_default())
|
||||
.and_then(|value| value.to_str().ok().map(|s| s.to_string()))
|
||||
})
|
||||
})
|
||||
{
|
||||
match alg.as_str() {
|
||||
ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str,
|
||||
ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str,
|
||||
ChecksumAlgorithm::SHA1 => checksum_sha1 = checksum_str,
|
||||
ChecksumAlgorithm::SHA256 => checksum_sha256 = checksum_str,
|
||||
ChecksumAlgorithm::CRC64NVME => checksum_crc64nvme = checksum_str,
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
// XXHash3/64/128 and SHA-512 have no typed UploadPartOutput field; echo the
|
||||
// server-computed part checksum as a raw response header (#1261).
|
||||
let upload_part_extra_checksum_headers = crate::app::object_usecase::additional_checksum_echo_pairs(&opts.want_checksum);
|
||||
|
||||
let output = UploadPartOutput {
|
||||
server_side_encryption: requested_sse,
|
||||
ssekms_key_id: requested_kms_key_id,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key_md5,
|
||||
checksum_crc32,
|
||||
checksum_crc32c,
|
||||
checksum_sha1,
|
||||
checksum_sha256,
|
||||
checksum_crc64nvme,
|
||||
e_tag: info.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut response = S3Response::new(output);
|
||||
crate::app::object_usecase::inject_additional_checksum_headers(
|
||||
&mut response.headers,
|
||||
&upload_part_extra_checksum_headers,
|
||||
);
|
||||
Ok(response)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
use super::*;
|
||||
|
||||
fn presigned_checksum_request(query: &str) -> S3Request<UploadPartInput> {
|
||||
let mut req = build_request(UploadPartInput::default(), Method::PUT);
|
||||
req.uri = format!("/bucket/object?{query}").parse().unwrap();
|
||||
req.extensions.insert(VerifiedPresignedRequest);
|
||||
req
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn normalize_presigned_part_checksums_validates_body() {
|
||||
// SHA256("abc"), including percent-encoded base64 punctuation.
|
||||
let checksum = "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=";
|
||||
let query =
|
||||
"x-amz-checksum-sha256=ungWv48Bz%2BpBQUDeXa4iI7ADYaOWF3qctBD%2FYfIAFa0%3D&x-amz-sdk-checksum-algorithm=SHA256";
|
||||
for payload in [b"abc", b"abd"] {
|
||||
let mut req = presigned_checksum_request(query);
|
||||
normalize_presigned_part_checksums(&mut req).unwrap();
|
||||
assert_eq!(req.input.checksum_sha256.as_deref(), Some(checksum));
|
||||
assert_eq!(
|
||||
req.input.checksum_algorithm.as_ref().map(|algorithm| algorithm.as_str()),
|
||||
Some(ChecksumAlgorithm::SHA256)
|
||||
);
|
||||
let mut reader = HashReader::from_stream(Cursor::new(payload), 3, 3, None, None, false).unwrap();
|
||||
reader.add_checksum(&req.headers, None, false).unwrap();
|
||||
assert_eq!(reader.content_crc_type(), Some(rustfs_rio::ChecksumType::SHA256));
|
||||
let mut bytes = Vec::new();
|
||||
let result = reader.read_to_end(&mut bytes).await;
|
||||
if payload == b"abc" {
|
||||
result.unwrap();
|
||||
assert_eq!(bytes, payload);
|
||||
assert_eq!(reader.content_crc().get("SHA256").map(String::as_str), Some(checksum));
|
||||
} else {
|
||||
let err = S3Error::from(ApiError::from(result.unwrap_err()));
|
||||
assert_eq!(*err.code(), S3ErrorCode::BadDigest);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn normalize_presigned_part_checksums_accepts_sdk_crc32_default() {
|
||||
let mut req = presigned_checksum_request("x-amz-checksum-crc32=y%2FQ5Jg%3D%3D&x-amz-sdk-checksum-algorithm=CRC32");
|
||||
normalize_presigned_part_checksums(&mut req).unwrap();
|
||||
assert_eq!(req.input.checksum_crc32.as_deref(), Some("y/Q5Jg=="));
|
||||
let mut reader = HashReader::from_stream(Cursor::new(b"123456789"), 9, 9, None, None, false).unwrap();
|
||||
reader.add_checksum(&req.headers, None, false).unwrap();
|
||||
let mut bytes = Vec::new();
|
||||
reader.read_to_end(&mut bytes).await.unwrap();
|
||||
assert_eq!(bytes, b"123456789");
|
||||
assert_eq!(reader.content_crc().get("CRC32").map(String::as_str), Some("y/Q5Jg=="));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_presigned_part_checksums_rejects_ambiguous_values() {
|
||||
for query in [
|
||||
"x-amz-checksum-sha256=one&x-amz-checksum-sha256=two",
|
||||
"x-amz-sdk-checksum-algorithm=SHA256&x-amz-sdk-checksum-algorithm=CRC32",
|
||||
] {
|
||||
let mut req = presigned_checksum_request(query);
|
||||
assert_eq!(
|
||||
*normalize_presigned_part_checksums(&mut req).unwrap_err().code(),
|
||||
S3ErrorCode::InvalidRequest
|
||||
);
|
||||
}
|
||||
let mut req = presigned_checksum_request("x-amz-checksum-sha256=one");
|
||||
req.headers.insert("x-amz-checksum-sha256", HeaderValue::from_static("two"));
|
||||
assert_eq!(
|
||||
*normalize_presigned_part_checksums(&mut req).unwrap_err().code(),
|
||||
S3ErrorCode::InvalidRequest
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_presigned_part_checksums_preserves_headers_and_rejects_invalid_values() {
|
||||
let mut req = presigned_checksum_request("partNumber=1&uploadId=test");
|
||||
req.input.checksum_sha256 = Some("existing".to_owned());
|
||||
req.headers
|
||||
.insert("x-amz-checksum-sha256", HeaderValue::from_static("existing"));
|
||||
normalize_presigned_part_checksums(&mut req).unwrap();
|
||||
assert_eq!(req.input.checksum_sha256.as_deref(), Some("existing"));
|
||||
assert_eq!(req.headers["x-amz-checksum-sha256"], "existing");
|
||||
|
||||
let mut req = presigned_checksum_request("x-amz-checksum-sha256=%0D%0A");
|
||||
assert_eq!(
|
||||
*normalize_presigned_part_checksums(&mut req).unwrap_err().code(),
|
||||
S3ErrorCode::InvalidArgument
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_presigned_part_checksums_requires_verified_presign() {
|
||||
let mut req = presigned_checksum_request("x-amz-checksum-sha256=one");
|
||||
req.extensions.remove::<VerifiedPresignedRequest>();
|
||||
normalize_presigned_part_checksums(&mut req).unwrap();
|
||||
assert!(req.input.checksum_sha256.is_none());
|
||||
assert!(req.headers.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_upload_part_size_uses_normalized_logical_length() {
|
||||
let result: rustfs_s3_types::S3Result<Option<i64>> = resolve_upload_part_size(Some(5242880), None);
|
||||
assert_eq!(result.expect("DTO length"), Some(5242880));
|
||||
assert_eq!(resolve_upload_part_size(None, None).expect("unknown length"), None);
|
||||
let body = StreamingBlob::from(Bytes::from_static(b"abc"));
|
||||
assert_eq!(resolve_upload_part_size(None, Some(&body)).expect("exact bytes"), Some(3));
|
||||
assert_eq!(resolve_upload_part_size(Some(0), Some(&body)).expect("explicit length wins"), Some(0));
|
||||
let body = StreamingBlob::wrap(futures::stream::iter([Ok::<_, std::io::Error>(Bytes::from_static(b"abc"))]));
|
||||
assert_eq!(futures::Stream::size_hint(&body), (1, Some(1)), "the stream knows its item count");
|
||||
assert_eq!(resolve_upload_part_size(None, Some(&body)).expect("unknown byte length"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_part_length_contract_rejects_unknown_and_all_negative_lengths() {
|
||||
let result: rustfs_s3_types::S3Result<i64> = require_upload_part_size(None, false);
|
||||
assert_eq!(
|
||||
result.expect_err("ordinary unknown").code(),
|
||||
&rustfs_s3_types::S3ErrorCode::MissingContentLength
|
||||
);
|
||||
assert_eq!(
|
||||
require_upload_part_size(None, true).expect_err("capped unknown").code(),
|
||||
&rustfs_s3_types::S3ErrorCode::UnexpectedContent
|
||||
);
|
||||
for capped in [false, true] {
|
||||
for size in [-1, -2, i64::MIN] {
|
||||
assert_eq!(
|
||||
require_upload_part_size(Some(size), capped).expect_err("negative").code(),
|
||||
&rustfs_s3_types::S3ErrorCode::UnexpectedContent
|
||||
);
|
||||
}
|
||||
assert_eq!(require_upload_part_size(Some(0), capped).expect("zero is valid"), 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_part_timeout_policy_preserves_disabled_and_capped_floor() {
|
||||
for seconds in [0, 1, 299, 300, 601] {
|
||||
let configured = Duration::from_secs(seconds);
|
||||
assert_eq!(upload_part_body_read_timeout(configured, false), configured);
|
||||
assert_eq!(upload_part_body_read_timeout(configured, true), Duration::from_secs(seconds.max(300)));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_upload_part_rejects_missing_body() {
|
||||
let input = UploadPartInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.part_number(1)
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::IncompleteBody);
|
||||
}
|
||||
|
||||
/// issue #7596: a part whose declared length exceeds the 5 GiB
|
||||
/// single-request ceiling is rejected before the body is polled or the
|
||||
/// store is consulted. Exact-cap and zero-length parts pass admission.
|
||||
#[tokio::test]
|
||||
async fn execute_upload_part_rejects_oversize_declared_part_before_reading_the_body() {
|
||||
let ceiling = i64::try_from(rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE).expect("ceiling fits i64");
|
||||
|
||||
for (declared, expect_too_large) in [(ceiling + 1, true), (ceiling, false), (0, false)] {
|
||||
let (body, polls) = crate::app::object::PollCountingBody::streaming_blob();
|
||||
let input = UploadPartInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.part_number(1)
|
||||
.body(Some(body))
|
||||
.content_length(Some(declared))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
if expect_too_large {
|
||||
assert_eq!(err.code(), &S3ErrorCode::EntityTooLarge, "declared {declared}");
|
||||
assert_eq!(polls.load(std::sync::atomic::Ordering::SeqCst), 0, "body must not be polled");
|
||||
} else {
|
||||
assert_ne!(err.code(), &S3ErrorCode::EntityTooLarge, "declared {declared} must pass admission");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_upload_part_rejects_invalid_part_number_before_body_lookup() {
|
||||
for part_number in [-1, 0, 10001] {
|
||||
let input = UploadPartInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.upload_id("upload-id".to_string())
|
||||
.part_number(part_number)
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = make_usecase().execute_upload_part(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_upload_part_rejects_unknown_length_before_admission_or_body_polling() {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations, MakeBucketOptions};
|
||||
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
|
||||
let bucket = format!("upload-part-length-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let concurrency_manager = Arc::new(ConcurrencyManager::with_large_put_admission_for_test(
|
||||
true,
|
||||
1,
|
||||
rustfs_config::DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES,
|
||||
Duration::ZERO,
|
||||
));
|
||||
let held = concurrency_manager
|
||||
.admit_multipart_part(1024)
|
||||
.await
|
||||
.expect("hold the only permit");
|
||||
let usecase = DefaultMultipartUsecase::with_context_and_concurrency_manager(Some(context), concurrency_manager);
|
||||
|
||||
for capped in [false, true] {
|
||||
let mut options = ObjectOptions::default();
|
||||
if capped {
|
||||
insert_str(&mut options.user_defined, SUFFIX_MAX_TOTAL_OBJECT_SIZE, "1024".to_owned());
|
||||
}
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, "object", &options)
|
||||
.await
|
||||
.expect("upload session");
|
||||
for content_length in [None, Some(-1)] {
|
||||
for declared_chunk_encoding in [false, true] {
|
||||
let (body, polls) = crate::app::object::PollCountingBody::streaming_blob();
|
||||
let input = UploadPartInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("object".to_owned())
|
||||
.upload_id(upload.upload_id.clone())
|
||||
.part_number(1)
|
||||
.content_length(content_length)
|
||||
.body(Some(body))
|
||||
.build()
|
||||
.expect("part request");
|
||||
let mut request = build_request(input, Method::PUT);
|
||||
request
|
||||
.headers
|
||||
.insert("x-amz-decoded-content-length", HeaderValue::from_static("1024"));
|
||||
if declared_chunk_encoding {
|
||||
request
|
||||
.headers
|
||||
.insert("content-encoding", HeaderValue::from_static("aws-chunked"));
|
||||
}
|
||||
let error = usecase
|
||||
.execute_upload_part(request)
|
||||
.await
|
||||
.expect_err("unknown or negative logical size");
|
||||
assert_eq!(
|
||||
error.code(),
|
||||
&if capped || content_length.is_some() {
|
||||
S3ErrorCode::UnexpectedContent
|
||||
} else {
|
||||
S3ErrorCode::MissingContentLength
|
||||
}
|
||||
);
|
||||
assert_eq!(polls.load(std::sync::atomic::Ordering::Relaxed), 0);
|
||||
}
|
||||
}
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, "object", &upload.upload_id, None, 1000, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("list rejected session");
|
||||
assert!(parts.parts.is_empty());
|
||||
}
|
||||
drop(held);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_upload_part_progress_timeout_releases_admission_and_leaves_no_part() {
|
||||
use crate::app::object::request_body::{BodyReadControl, ObservedBody};
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use http_body::Frame;
|
||||
use http_body_util::StreamBody;
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
crate::app::runtime_sources::install_test_app_context(Arc::clone(&store)).await;
|
||||
let ambient = crate::app::runtime_sources::current_app_context().expect("test app context");
|
||||
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
|
||||
let bucket = format!("part-progress-{}", Uuid::new_v4().simple());
|
||||
store.make_bucket(&bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
let manager = Arc::new(ConcurrencyManager::with_put_admission_for_test(true, 1, Duration::ZERO));
|
||||
let usecase = DefaultMultipartUsecase::with_context_and_concurrency_manager(Some(context), Arc::clone(&manager));
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel::<Result<Frame<Bytes>, std::io::Error>>();
|
||||
let control = BodyReadControl::default();
|
||||
let body = StreamingBlob::from(crate::app::storage_api::s3::Body::http_body_unsync(ObservedBody::new(
|
||||
StreamBody::new(UnboundedReceiverStream::new(receiver)),
|
||||
control.clone(),
|
||||
)));
|
||||
let input = UploadPartInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key("object".to_owned())
|
||||
.upload_id(upload.upload_id.clone())
|
||||
.part_number(1)
|
||||
.content_length(Some(65536))
|
||||
.body(Some(body))
|
||||
.build()
|
||||
.unwrap();
|
||||
let mut request = build_request(input, Method::PUT);
|
||||
request.extensions.insert(control.clone());
|
||||
let owner = tokio::spawn(async move { usecase.execute_upload_part(request).await });
|
||||
control.wait_until_pending_for_test().await;
|
||||
assert!(matches!(
|
||||
manager.admit_put_object(65536).await.unwrap(),
|
||||
ForegroundWriteAdmission::Rejected
|
||||
));
|
||||
control.advance_wait_for_test(Duration::from_secs(60));
|
||||
sender.send(Ok(Frame::data(Bytes::from_static(b"x")))).unwrap();
|
||||
assert_eq!(owner.await.unwrap().unwrap_err().code(), &S3ErrorCode::RequestTimeout);
|
||||
assert!(matches!(
|
||||
manager.admit_put_object(65536).await.unwrap(),
|
||||
ForegroundWriteAdmission::Admitted(_)
|
||||
));
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, "object", &upload.upload_id, None, 1000, &ObjectOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(parts.parts.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn execute_upload_part_rejects_when_foreground_write_admission_is_full() {
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations, MakeBucketOptions};
|
||||
|
||||
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
|
||||
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
|
||||
let bucket = format!("upload-part-admission-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create upload part admission test bucket");
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, "object", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create multipart upload");
|
||||
let concurrency_manager = Arc::new(ConcurrencyManager::with_large_put_admission_for_test(
|
||||
true,
|
||||
1,
|
||||
rustfs_config::DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES,
|
||||
Duration::ZERO,
|
||||
));
|
||||
let held = concurrency_manager
|
||||
.admit_multipart_part(1024)
|
||||
.await
|
||||
.expect("first multipart part should acquire the only permit");
|
||||
let usecase =
|
||||
DefaultMultipartUsecase::with_context_and_concurrency_manager(Some(context), Arc::clone(&concurrency_manager));
|
||||
let body = StreamingBlob::wrap(futures::stream::pending::<Result<Bytes, std::io::Error>>());
|
||||
let input = UploadPartInput::builder()
|
||||
.bucket(bucket)
|
||||
.key("object".to_string())
|
||||
.upload_id(upload.upload_id)
|
||||
.part_number(1)
|
||||
.content_length(Some(1024))
|
||||
.body(Some(body))
|
||||
.build()
|
||||
.expect("upload part input should build");
|
||||
|
||||
let err = usecase
|
||||
.execute_upload_part(build_request(input, Method::PUT))
|
||||
.await
|
||||
.expect_err("full foreground write admission should reject before body ingest");
|
||||
assert_eq!(err.code(), &S3ErrorCode::SlowDown);
|
||||
drop(held);
|
||||
}
|
||||
|
||||
mod body_read_tests;
|
||||
}
|
||||
@@ -0,0 +1,556 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! UploadPartCopy.
|
||||
|
||||
use super::*;
|
||||
|
||||
fn normalize_copy_source_version_id(version_id: Option<&str>) -> S3Result<Option<String>> {
|
||||
match version_id {
|
||||
Some(v) => {
|
||||
let trimmed = v.trim();
|
||||
if trimmed.eq_ignore_ascii_case("null") {
|
||||
Ok(Some(Uuid::nil().to_string()))
|
||||
} else if Uuid::parse_str(trimmed).is_ok() {
|
||||
Ok(Some(trimmed.to_string()))
|
||||
} else {
|
||||
Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidArgument,
|
||||
"Invalid version id specified in copy source",
|
||||
))
|
||||
}
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn format_copy_source_version_id(version_id: Option<Uuid>, src_versioned: bool, src_version_suspended: bool) -> Option<String> {
|
||||
if src_versioned || src_version_suspended {
|
||||
Some(match version_id {
|
||||
Some(vid) if vid != Uuid::nil() => vid.to_string(),
|
||||
_ => "null".to_string(),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns InvalidRange error if CopySourceRange end exceeds the source object size.
|
||||
/// Used by execute_upload_part_copy to reject out-of-bounds ranges per S3 spec.
|
||||
fn validate_copy_source_range_not_exceeds(range_spec: &HTTPRangeSpec, object_size: i64) -> S3Result<()> {
|
||||
if range_spec.end >= object_size {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidRange,
|
||||
"The requested range is not satisfiable".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl DefaultMultipartUsecase {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
#[hotpath::measure(impl_type = "MultipartUsecase")]
|
||||
pub async fn execute_upload_part_copy(
|
||||
&self,
|
||||
req: S3Request<UploadPartCopyInput>,
|
||||
) -> S3Result<S3Response<UploadPartCopyOutput>> {
|
||||
reject_presigned_multipart_max_total_object_size_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedSigV4Request>().is_some(),
|
||||
)?;
|
||||
reject_presigned_put_max_content_length_for_other_operation(
|
||||
&req.headers,
|
||||
req.uri.query(),
|
||||
req.extensions.get::<VerifiedPresignedRequest>().is_some(),
|
||||
)?;
|
||||
// Captured before `req.input` is destructured below.
|
||||
let copy_principal = SseKmsPrincipal::from_request(&req);
|
||||
let source_bucket = match &req.input.copy_source {
|
||||
CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)),
|
||||
CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)),
|
||||
CopySource::Bucket { bucket, .. } => bucket.to_string(),
|
||||
};
|
||||
let mut source_generation_opts = ObjectOptions::default();
|
||||
apply_copy_source_bucket_generation_guard(&req, &source_bucket, &mut source_generation_opts)?;
|
||||
let expected_source_incarnation_id = source_generation_opts.expected_bucket_incarnation_id;
|
||||
let mut destination_generation_opts = ObjectOptions::default();
|
||||
apply_bucket_generation_guard(&req, &req.input.bucket, &mut destination_generation_opts)?;
|
||||
let expected_destination_incarnation_id = destination_generation_opts.expected_bucket_incarnation_id;
|
||||
let UploadPartCopyInput {
|
||||
bucket,
|
||||
key,
|
||||
copy_source,
|
||||
copy_source_range,
|
||||
part_number,
|
||||
upload_id,
|
||||
copy_source_if_match,
|
||||
copy_source_if_none_match,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key,
|
||||
sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5,
|
||||
..
|
||||
} = req.input;
|
||||
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
|
||||
let (src_bucket, src_key, raw_src_version_id) = match copy_source {
|
||||
CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)),
|
||||
CopySource::Outpost { .. } => return Err(s3_error!(NotImplemented)),
|
||||
CopySource::Bucket {
|
||||
bucket: ref src_bucket,
|
||||
key: ref src_key,
|
||||
version_id,
|
||||
} => (src_bucket.to_string(), src_key.to_string(), version_id.map(|v| v.to_string())),
|
||||
};
|
||||
let src_version_id = normalize_copy_source_version_id(raw_src_version_id.as_deref())?;
|
||||
|
||||
let rs = if let Some(range_str) = copy_source_range {
|
||||
Some(parse_copy_source_range(&range_str)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let part_id = parse_upload_part_number(part_number)?;
|
||||
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
let (
|
||||
source_bucket_lifecycle_guard,
|
||||
current_source_incarnation_id,
|
||||
destination_bucket_lifecycle_guard_storage,
|
||||
current_destination_incarnation_id,
|
||||
) = store
|
||||
.acquire_copy_bucket_incarnation_fences(
|
||||
&src_bucket,
|
||||
expected_source_incarnation_id,
|
||||
&bucket,
|
||||
expected_destination_incarnation_id,
|
||||
)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let destination_bucket_lifecycle_guard = destination_bucket_lifecycle_guard_storage
|
||||
.as_ref()
|
||||
.unwrap_or(&source_bucket_lifecycle_guard);
|
||||
if source_bucket_lifecycle_guard.is_lock_lost() || destination_bucket_lifecycle_guard.is_lock_lost() {
|
||||
return Err(ApiError::from(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "copy_bucket_generation",
|
||||
bucket: bucket.clone(),
|
||||
object: key.clone(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
})
|
||||
.into());
|
||||
}
|
||||
let mut dst_opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(current_destination_incarnation_id),
|
||||
..Default::default()
|
||||
};
|
||||
if src_bucket != bucket {
|
||||
source_bucket_lifecycle_guard.attach_to_object_options(&mut dst_opts);
|
||||
}
|
||||
destination_bucket_lifecycle_guard.attach_to_object_options(&mut dst_opts);
|
||||
|
||||
let mp_info = store
|
||||
.get_multipart_info(&bucket, &key, &upload_id, &dst_opts)
|
||||
.await
|
||||
.map_err(multipart_upload_api_error)?;
|
||||
authorize_sse_kms_multipart_write(copy_principal.as_ref(), &mp_info.user_defined).await?;
|
||||
let destination_size_limit = multipart_max_total_object_size(&mp_info.user_defined)?;
|
||||
EncryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
server_side_encryption: None,
|
||||
ssekms_key_id: None,
|
||||
ssekms_context: None,
|
||||
sse_customer_algorithm: sse_customer_algorithm.clone(),
|
||||
sse_customer_key: sse_customer_key.clone(),
|
||||
sse_customer_key_md5: sse_customer_key_md5.clone(),
|
||||
content_size: 0,
|
||||
principal: None,
|
||||
}
|
||||
.validate_multipart_ssec(&mp_info.user_defined)?;
|
||||
|
||||
let src_versioned = BucketVersioningSys::prefix_enabled(&src_bucket, &src_key).await;
|
||||
let src_version_suspended = BucketVersioningSys::prefix_suspended(&src_bucket, &src_key).await;
|
||||
let mut src_opts = copy_src_opts(&src_bucket, &src_key, &req.headers).map_err(ApiError::from)?;
|
||||
src_opts.version_id = src_version_id.clone();
|
||||
src_opts.versioned = src_versioned;
|
||||
src_opts.version_suspended = src_version_suspended;
|
||||
|
||||
let h = build_ssec_read_headers(
|
||||
copy_source_sse_customer_algorithm.as_ref(),
|
||||
copy_source_sse_customer_key.as_ref(),
|
||||
copy_source_sse_customer_key_md5.as_ref(),
|
||||
);
|
||||
let get_opts = ObjectOptions {
|
||||
version_id: src_opts.version_id.clone(),
|
||||
versioned: src_opts.versioned,
|
||||
version_suspended: src_opts.version_suspended,
|
||||
expected_bucket_incarnation_id: Some(current_source_incarnation_id),
|
||||
..Default::default()
|
||||
};
|
||||
if source_bucket_lifecycle_guard.is_lock_lost() {
|
||||
return Err(ApiError::from(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "copy_source_bucket_generation",
|
||||
bucket: src_bucket.clone(),
|
||||
object: src_key.clone(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
})
|
||||
.into());
|
||||
}
|
||||
|
||||
let prepared_source = store
|
||||
.prepare_get_object_reader(&src_bucket, &src_key, rs.clone(), h, &get_opts)
|
||||
.await
|
||||
.map_err(map_get_object_reader_error)?;
|
||||
|
||||
let src_info = prepared_source.object_info();
|
||||
if src_info.delete_marker {
|
||||
let error = if src_opts.version_id.is_none() {
|
||||
StorageError::ObjectNotFound(src_bucket.clone(), src_key.clone())
|
||||
} else {
|
||||
StorageError::MethodNotAllowed
|
||||
};
|
||||
return Err(map_get_object_reader_error(error).into());
|
||||
}
|
||||
|
||||
// Same shape as CopyObject: the part copy reads the source plaintext, and the source
|
||||
// read resolves its material inside the object layer, which carries no request identity.
|
||||
authorize_sse_kms_object_read(copy_principal.as_ref(), &src_info.user_defined).await?;
|
||||
|
||||
let resolved_src_version_id = format_copy_source_version_id(src_info.version_id, src_versioned, src_version_suspended);
|
||||
|
||||
if let Some(if_match) = copy_source_if_match {
|
||||
if let Some(ref etag) = src_info.etag {
|
||||
if let Some(strong_etag) = if_match.into_etag() {
|
||||
if ETag::Strong(etag.clone()) != strong_etag {
|
||||
return Err(s3_error!(PreconditionFailed));
|
||||
}
|
||||
} else {
|
||||
return Err(s3_error!(PreconditionFailed));
|
||||
}
|
||||
} else {
|
||||
return Err(s3_error!(PreconditionFailed));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(if_none_match) = copy_source_if_none_match
|
||||
&& let Some(ref etag) = src_info.etag
|
||||
&& let Some(strong_etag) = if_none_match.into_etag()
|
||||
&& ETag::Strong(etag.clone()) == strong_etag
|
||||
{
|
||||
return Err(s3_error!(PreconditionFailed));
|
||||
}
|
||||
|
||||
let source_actual_size = src_info.get_actual_size().ok().filter(|size| *size >= 0);
|
||||
let source_logical_size = match source_actual_size {
|
||||
Some(size) => size,
|
||||
None if destination_size_limit.is_some() => {
|
||||
return Err(S3Error::new(S3ErrorCode::UnexpectedContent));
|
||||
}
|
||||
None => src_info.size,
|
||||
};
|
||||
|
||||
let (_start_offset, length) = if let Some(ref range_spec) = rs {
|
||||
// Copy-source ranges are expressed over the logical plaintext object.
|
||||
// Encrypted (and compressed) objects have a larger or smaller physical
|
||||
// representation, so validating against `size` rejects valid later parts.
|
||||
validate_copy_source_range_not_exceeds(range_spec, source_logical_size)?;
|
||||
|
||||
range_spec
|
||||
.get_offset_length(source_logical_size)
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InvalidRange, e.to_string()))?
|
||||
} else {
|
||||
(0, source_logical_size)
|
||||
};
|
||||
|
||||
// Resolve the range under the prepared source's read lock, then try the
|
||||
// shared budget without waiting while holding namespace/lifecycle locks.
|
||||
// Keep both the budget and source cancellation alive through the write.
|
||||
let admission_length = source_actual_size.map_or(-1, |_| length);
|
||||
let _copy_admission = match self.concurrency_manager().try_admit_multipart_part_copy(admission_length) {
|
||||
ForegroundWriteAdmission::Disabled => None,
|
||||
ForegroundWriteAdmission::Admitted(permit) => Some(permit),
|
||||
ForegroundWriteAdmission::Rejected => {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::SlowDown,
|
||||
"foreground write concurrency limit reached, please reduce your request rate",
|
||||
));
|
||||
}
|
||||
};
|
||||
let (src_reader, _source_cancellation) = prepared_source
|
||||
.into_reader_for_copy()
|
||||
.await
|
||||
.map_err(map_get_object_reader_error)?;
|
||||
let src_stream = src_reader.stream;
|
||||
|
||||
let is_disk_compressed =
|
||||
rustfs_utils::http::contains_key_str(&mp_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
|
||||
|
||||
let actual_size = length;
|
||||
|
||||
let mut write_plan = WritePlan::new();
|
||||
let mut reader = if is_disk_compressed {
|
||||
let algorithm = CompressionAlgorithm::default();
|
||||
let hrd = HashReader::from_stream(src_stream, length, actual_size, None, None, false).map_err(ApiError::from)?;
|
||||
write_plan = write_plan.with_compression(algorithm);
|
||||
hrd
|
||||
} else {
|
||||
HashReader::from_stream(src_stream, length, actual_size, None, None, false).map_err(ApiError::from)?
|
||||
};
|
||||
|
||||
let server_side_encryption = mp_info
|
||||
.user_defined
|
||||
.get("x-amz-server-side-encryption")
|
||||
.map(|s| {
|
||||
ServerSideEncryption::from_str(s)
|
||||
.map_err(|e| ApiError::from(StorageError::other(format!("Invalid server-side encryption: {e}"))))
|
||||
})
|
||||
.transpose()?;
|
||||
let has_ssec = mp_info
|
||||
.user_defined
|
||||
.contains_key("x-amz-server-side-encryption-customer-algorithm");
|
||||
let ssekms_key_id = match server_side_encryption.as_ref() {
|
||||
Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => mp_info
|
||||
.user_defined
|
||||
.get("x-amz-server-side-encryption-aws-kms-key-id")
|
||||
.map(|s| s.to_string()),
|
||||
_ => None,
|
||||
};
|
||||
let (requested_sse, requested_kms_key_id, dst_user_defined) = if has_ssec {
|
||||
let ssec_material = sse_decryption(DecryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
metadata: &mp_info.user_defined,
|
||||
sse_customer_key: sse_customer_key.as_ref(),
|
||||
sse_customer_key_md5: sse_customer_key_md5.as_ref(),
|
||||
principal: None,
|
||||
})
|
||||
.await?
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing SSE-C session material")))?;
|
||||
let ssec_write = match ssec_material.key_kind {
|
||||
EncryptionKeyKind::Object => WriteEncryption::multipart_object_key(ssec_material.key_bytes, part_id as u32),
|
||||
EncryptionKeyKind::Direct => {
|
||||
WriteEncryption::multipart(ssec_material.key_bytes, ssec_material.base_nonce, part_id)
|
||||
}
|
||||
};
|
||||
write_plan = write_plan.with_encryption(ssec_write);
|
||||
(
|
||||
Some(ssec_material.server_side_encryption),
|
||||
ssec_material.kms_key_id,
|
||||
mp_info.user_defined.clone(),
|
||||
)
|
||||
} else if let Some(server_side_encryption) = server_side_encryption {
|
||||
// Reuse the destination envelope after authorizing this caller against
|
||||
// its pinned key above. The source read is separately authorized.
|
||||
let managed_material = sse_decryption(DecryptionRequest {
|
||||
bucket: &bucket,
|
||||
key: &key,
|
||||
metadata: &mp_info.user_defined,
|
||||
sse_customer_key: None,
|
||||
sse_customer_key_md5: None,
|
||||
principal: None,
|
||||
})
|
||||
.await?
|
||||
.ok_or_else(|| ApiError::from(StorageError::other("Missing managed SSE session material")))?;
|
||||
let managed_write = match managed_material.key_kind {
|
||||
EncryptionKeyKind::Object => WriteEncryption::multipart_object_key(managed_material.key_bytes, part_id as u32),
|
||||
EncryptionKeyKind::Direct => {
|
||||
WriteEncryption::multipart(managed_material.key_bytes, managed_material.base_nonce, part_id)
|
||||
}
|
||||
};
|
||||
write_plan = write_plan.with_encryption(managed_write);
|
||||
(Some(server_side_encryption), ssekms_key_id, mp_info.user_defined.clone())
|
||||
} else {
|
||||
(None, None, mp_info.user_defined.clone())
|
||||
};
|
||||
|
||||
reader = write_plan.apply(reader, actual_size).map_err(ApiError::from)?;
|
||||
|
||||
if let Some(checksum_algorithm) = mp_info
|
||||
.user_defined
|
||||
.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)
|
||||
.filter(|checksum_algorithm| !checksum_algorithm.is_empty())
|
||||
{
|
||||
let checksum_type = rustfs_rio::ChecksumType::from_string_with_obj_type(
|
||||
checksum_algorithm,
|
||||
mp_info
|
||||
.user_defined
|
||||
.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE)
|
||||
.map(String::as_str)
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
if !checksum_type.is_set() {
|
||||
return Err(ApiError::from(StorageError::other(format!(
|
||||
"Invalid multipart checksum type: {checksum_algorithm}"
|
||||
)))
|
||||
.into());
|
||||
}
|
||||
reader.add_calculated_checksum(checksum_type).map_err(ApiError::from)?;
|
||||
}
|
||||
|
||||
let mut reader = PutObjReader::new(reader);
|
||||
|
||||
dst_opts.user_defined = dst_user_defined;
|
||||
|
||||
let part_info = store
|
||||
.put_object_part(&bucket, &key, &upload_id, part_id, &mut reader, &dst_opts)
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let copy_checksums = reader.as_hash_reader().content_crc();
|
||||
let checksum_value = |checksum_type: rustfs_rio::ChecksumType| copy_checksums.get(&checksum_type.to_string()).cloned();
|
||||
|
||||
let copy_part_result = CopyPartResult {
|
||||
checksum_crc32: checksum_value(rustfs_rio::ChecksumType::CRC32),
|
||||
checksum_crc32c: checksum_value(rustfs_rio::ChecksumType::CRC32C),
|
||||
checksum_sha1: checksum_value(rustfs_rio::ChecksumType::SHA1),
|
||||
checksum_sha256: checksum_value(rustfs_rio::ChecksumType::SHA256),
|
||||
checksum_crc64nvme: checksum_value(rustfs_rio::ChecksumType::CRC64_NVME),
|
||||
e_tag: part_info.etag.map(|etag| to_s3s_xml_etag(&etag)),
|
||||
last_modified: part_info.last_mod.map(Timestamp::from),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let output = UploadPartCopyOutput {
|
||||
copy_part_result: Some(copy_part_result),
|
||||
copy_source_version_id: resolved_src_version_id,
|
||||
server_side_encryption: requested_sse,
|
||||
ssekms_key_id: requested_kms_key_id,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key_md5,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_upload_part_copy_returns_internal_error_when_store_uninitialized() {
|
||||
let input = UploadPartCopyInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: "src-bucket".into(),
|
||||
key: "src-object".into(),
|
||||
version_id: None,
|
||||
})
|
||||
.part_number(1)
|
||||
.upload_id("upload-id".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = Box::pin(make_usecase().execute_upload_part_copy(req)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_upload_part_copy_rejects_invalid_part_number_before_store_lookup() {
|
||||
for part_number in [-1, 0, 10001] {
|
||||
let input = UploadPartCopyInput::builder()
|
||||
.bucket("bucket".to_string())
|
||||
.key("object".to_string())
|
||||
.copy_source(CopySource::Bucket {
|
||||
bucket: "src-bucket".into(),
|
||||
key: "src-object".into(),
|
||||
version_id: None,
|
||||
})
|
||||
.part_number(part_number)
|
||||
.upload_id("upload-id".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request(input, Method::PUT);
|
||||
|
||||
let err = Box::pin(make_usecase().execute_upload_part_copy(req)).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("partNumber must be between 1 and 10000"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_copy_source_range_not_exceeds_returns_invalid_range_when_range_exceeds() {
|
||||
use super::validate_copy_source_range_not_exceeds;
|
||||
|
||||
let range_exceeds = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: 0,
|
||||
end: 21,
|
||||
};
|
||||
let err = validate_copy_source_range_not_exceeds(&range_exceeds, 5).unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRange);
|
||||
assert!(err.to_string().contains("not satisfiable"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_copy_source_range_not_exceeds_ok_when_range_valid() {
|
||||
use super::validate_copy_source_range_not_exceeds;
|
||||
|
||||
let range_valid = HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: 0,
|
||||
end: 4,
|
||||
};
|
||||
assert!(validate_copy_source_range_not_exceeds(&range_valid, 5).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_copy_source_range_not_exceeds_ok_for_suffix_range() {
|
||||
use super::validate_copy_source_range_not_exceeds;
|
||||
|
||||
let range_suffix = HTTPRangeSpec {
|
||||
is_suffix_length: true,
|
||||
start: -5,
|
||||
end: -1,
|
||||
};
|
||||
assert!(validate_copy_source_range_not_exceeds(&range_suffix, 5).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_part_copy_normalizes_and_formats_source_version_id() {
|
||||
assert_eq!(normalize_copy_source_version_id(None).expect("missing version id"), None);
|
||||
assert_eq!(
|
||||
normalize_copy_source_version_id(Some(" null ")).expect("null version id"),
|
||||
Some(Uuid::nil().to_string())
|
||||
);
|
||||
let valid = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
normalize_copy_source_version_id(Some(&format!(" {valid} "))).expect("valid uuid"),
|
||||
Some(valid.to_string())
|
||||
);
|
||||
let err = normalize_copy_source_version_id(Some("not-a-uuid")).expect_err("invalid uuid should fail");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
|
||||
assert_eq!(format_copy_source_version_id(Some(Uuid::nil()), false, false), None);
|
||||
assert_eq!(format_copy_source_version_id(None, true, false), Some("null".to_string()));
|
||||
assert_eq!(format_copy_source_version_id(Some(Uuid::nil()), false, true), Some("null".to_string()));
|
||||
assert_eq!(format_copy_source_version_id(Some(valid), true, false), Some(valid.to_string()));
|
||||
}
|
||||
|
||||
mod copy_admission_tests;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -53,16 +53,10 @@ pub(crate) mod s3 {
|
||||
PersistedReplicationFilter, PersistedReplicationRule, PersistedReplicationTag,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::access::S3Access;
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::auth::{Credentials, SimpleAuth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::config::{S3Config, StaticConfigProvider};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::dto::{
|
||||
BucketLocationConstraint, CreateBucketConfiguration, CreateBucketInput, CreateMultipartUploadInput, DeleteObjectInput,
|
||||
ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, PutObjectInput, Tag, Tagging,
|
||||
};
|
||||
pub(crate) use s3s::dto;
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::dto::{GetObjectInput, HeadObjectInput, ListObjectsV2Input, ListObjectsV2Output};
|
||||
#[cfg(test)]
|
||||
@@ -72,6 +66,8 @@ pub(crate) mod s3 {
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::stream::upload_stream::{UploadStream, UploadStreamError};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::xml;
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::xml::{Serialize as XmlSerialize, Serializer as XmlSerializer};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::{Body, BodySizeLimitExceeded, S3, S3Response};
|
||||
@@ -317,11 +313,10 @@ pub(crate) mod access {
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::ReqInfo;
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{
|
||||
ListBucketsAuthzRequest, PostObjectRequestMarker, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess,
|
||||
TableDataPlaneListCursorPosition, TableDataPlanePublicationGuards, apply_bucket_generation_guard,
|
||||
apply_copy_source_bucket_generation_guard, authorize_copy_object_tags, authorize_request,
|
||||
bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header,
|
||||
load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
|
||||
PostObjectRequestMarker, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, TableDataPlaneListCursorPosition,
|
||||
TableDataPlanePublicationGuards, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard,
|
||||
authorize_copy_object_tags, authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action,
|
||||
has_bypass_governance_header, load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
|
||||
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller,
|
||||
replication_proxy_allowed, replication_request_authorized, req_info_mut, req_info_ref,
|
||||
retained_table_data_plane_publication_guards, set_requested_object_lock_retain_until,
|
||||
@@ -1673,9 +1668,9 @@ pub(crate) mod timeout_wrapper {
|
||||
pub(crate) mod s3_api {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use crate::storage::storage_api::s3_api_consumer::bucket::{
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_object_versions_output, build_list_objects_output,
|
||||
build_list_objects_v2_output, encode_list_objects_v2_continuation_token, parse_list_object_versions_params,
|
||||
parse_list_objects_v2_params,
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
|
||||
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
|
||||
parse_list_object_versions_params, parse_list_objects_v2_params,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::s3_api_consumer::common::rustfs_owner;
|
||||
}
|
||||
@@ -1711,7 +1706,7 @@ pub(crate) mod bucket_usecase {
|
||||
pub(crate) mod contract {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use super::super::super::storage_contracts::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
|
||||
BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1823,30 +1818,14 @@ pub(crate) mod select_object {
|
||||
};
|
||||
}
|
||||
|
||||
/// The storage items the owned/legacy authorization parity tests drive.
|
||||
#[cfg(test)]
|
||||
pub(crate) mod owned_parity {
|
||||
pub(crate) use crate::storage::storage_api::owned_parity_consumer::{
|
||||
DenialAuditor, DenialRateLimiter, DenialScope, FS, ObjectTagConditions, PostObjectPolicyHeaders, PostObjectRequestMarker,
|
||||
ReqInfo, RequestContext, RequestedCopySource, RequestedObjectLegalHold, RequestedObjectLockRetainUntil,
|
||||
RequestedObjectLockRetention, RequestedObjectTags, SUPPRESSED_DENIALS_TAG, TableDataPlanePublicationGuards,
|
||||
authorization_denial_entry, authorization_denial_entry_for_view, authorize_request, parity_test_support,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod context {
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::EndpointServerPools;
|
||||
pub(crate) use super::bucket;
|
||||
pub(crate) use super::runtime;
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{
|
||||
AuditRequestView, AuthorizationRequest, DenialAuditor, DenialScope, authorize_request_core,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{
|
||||
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, MultipartStagingOperation,
|
||||
ObjectTagConditions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectPolicyHeaders,
|
||||
PostObjectRequestMarker, ReqInfo, RequestedCopySource, RequestedObjectLegalHold, RequestedObjectLockRetainUntil,
|
||||
RequestedObjectLockRetention, RequestedObjectTags, StagedMultipartPartAuthorization, TableDataPlaneListAccess,
|
||||
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, ObjectTagConditions,
|
||||
OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectRequestMarker, ReqInfo, TableDataPlaneListAccess,
|
||||
TableDataPlanePublicationGuards,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::request_context_consumer::RequestContext;
|
||||
|
||||
@@ -78,7 +78,7 @@ impl FromStr for S3Stack {
|
||||
impl S3Stack {
|
||||
/// Unset selects the default; a set but invalid value, including an empty
|
||||
/// one, is a startup error.
|
||||
pub(crate) fn from_env() -> std::io::Result<Self> {
|
||||
fn from_env() -> std::io::Result<Self> {
|
||||
match std::env::var_os(ENV_RUSTFS_S3_STACK) {
|
||||
None => Ok(Self::default()),
|
||||
Some(value) => value
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Test-only source reader for text guards that must survive a mechanical file split.
|
||||
//!
|
||||
//! A text guard asserts that some literal appears in production source. When it reads one file
|
||||
//! with `include_str!`, moving the guarded function into another file either breaks the guard or,
|
||||
//! worse, makes it stop seeing the code it guards. A guard built on [`SourceSet`] instead names
|
||||
//! the file it reads today plus the sibling directory a split moves that file's code into, and
|
||||
//! reads every `.rs` file of both at test time.
|
||||
//!
|
||||
//! The named file is required: a missing file panics with its path, so a guard never passes on
|
||||
//! absent input. The directory is optional while the split has not happened; once it exists,
|
||||
//! every `.rs` file under it (recursively) is read in byte order of its manifest-relative path,
|
||||
//! after the named file.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// One source file of a [`SourceSet`].
|
||||
pub(crate) struct SourceFile {
|
||||
/// Path relative to the `rustfs` package root, with `/` separators.
|
||||
pub(crate) path: String,
|
||||
pub(crate) text: String,
|
||||
}
|
||||
|
||||
/// The ordered files a text guard reads: one required file, then the split directory.
|
||||
pub(crate) struct SourceSet {
|
||||
files: Vec<SourceFile>,
|
||||
}
|
||||
|
||||
impl SourceSet {
|
||||
/// Reads `file` and every `.rs` file under `split_dir`, both relative to the `rustfs`
|
||||
/// package root.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics, naming the path, when `file` is missing or unreadable, when `split_dir` exists but
|
||||
/// is not a directory, or when a file under `split_dir` cannot be read.
|
||||
pub(crate) fn read(file: &str, split_dir: &str) -> Self {
|
||||
Self::read_under(Path::new(env!("CARGO_MANIFEST_DIR")), file, split_dir)
|
||||
}
|
||||
|
||||
fn read_under(root: &Path, file: &str, split_dir: &str) -> Self {
|
||||
let required = root.join(file);
|
||||
let text = std::fs::read_to_string(&required)
|
||||
.unwrap_or_else(|err| panic!("guard source {} must exist and be readable: {err}", required.display()));
|
||||
let mut files = vec![SourceFile {
|
||||
path: file.to_owned(),
|
||||
text,
|
||||
}];
|
||||
|
||||
let dir = root.join(split_dir);
|
||||
if dir.exists() {
|
||||
assert!(dir.is_dir(), "guard split path {} must be a directory", dir.display());
|
||||
let mut paths = Vec::new();
|
||||
collect_rust_files(&dir, &mut paths);
|
||||
let mut split_files: Vec<SourceFile> = paths
|
||||
.into_iter()
|
||||
.map(|path| {
|
||||
let text = std::fs::read_to_string(&path)
|
||||
.unwrap_or_else(|err| panic!("guard source {} must be readable: {err}", path.display()));
|
||||
let relative = path
|
||||
.strip_prefix(root)
|
||||
.unwrap_or_else(|_| panic!("guard source {} must be under {}", path.display(), root.display()));
|
||||
let path = relative
|
||||
.components()
|
||||
.map(|component| component.as_os_str().to_string_lossy())
|
||||
.collect::<Vec<_>>()
|
||||
.join("/");
|
||||
SourceFile { path, text }
|
||||
})
|
||||
.collect();
|
||||
split_files.sort_by(|left, right| left.path.cmp(&right.path));
|
||||
files.extend(split_files);
|
||||
}
|
||||
|
||||
Self { files }
|
||||
}
|
||||
|
||||
/// Every file of the set, the required file first.
|
||||
pub(crate) fn files(&self) -> &[SourceFile] {
|
||||
&self.files
|
||||
}
|
||||
|
||||
/// Whether any one file of the set contains `needle`.
|
||||
pub(crate) fn contains(&self, needle: &str) -> bool {
|
||||
self.files.iter().any(|file| file.text.contains(needle))
|
||||
}
|
||||
|
||||
/// Non-overlapping occurrences of `needle`, summed over the files; a match never spans two
|
||||
/// files.
|
||||
pub(crate) fn count(&self, needle: &str) -> usize {
|
||||
self.files.iter().map(|file| file.text.matches(needle).count()).sum()
|
||||
}
|
||||
|
||||
/// The text after `marker` in the one file that contains it.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics unless `marker` occurs exactly once across the whole set.
|
||||
pub(crate) fn after_unique_marker(&self, marker: &str) -> &str {
|
||||
let occurrences = self.count(marker);
|
||||
assert_eq!(occurrences, 1, "marker {marker:?} must occur exactly once across {}", self.describe());
|
||||
self.files
|
||||
.iter()
|
||||
.find_map(|file| file.text.split_once(marker).map(|(_, rest)| rest))
|
||||
.unwrap_or_else(|| unreachable!("a counted marker is found"))
|
||||
}
|
||||
|
||||
/// Manifest-relative paths of the set, for panic messages.
|
||||
pub(crate) fn describe(&self) -> String {
|
||||
self.files
|
||||
.iter()
|
||||
.map(|file| file.path.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_rust_files(dir: &Path, out: &mut Vec<PathBuf>) {
|
||||
let entries =
|
||||
std::fs::read_dir(dir).unwrap_or_else(|err| panic!("guard split directory {} must be readable: {err}", dir.display()));
|
||||
for entry in entries {
|
||||
let entry = entry.unwrap_or_else(|err| panic!("guard split directory {} must be listable: {err}", dir.display()));
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
collect_rust_files(&path, out);
|
||||
} else if path.extension().is_some_and(|extension| extension == "rs") {
|
||||
out.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::SourceSet;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
fn write(root: &Path, relative: &str, text: &str) {
|
||||
let path = root.join(relative);
|
||||
fs::create_dir_all(path.parent().expect("fixture path has a parent")).expect("create fixture dir");
|
||||
fs::write(path, text).expect("write fixture");
|
||||
}
|
||||
|
||||
fn panic_message(run: impl FnOnce() + std::panic::UnwindSafe) -> String {
|
||||
let payload = std::panic::catch_unwind(run).expect_err("the guard input must be rejected");
|
||||
payload
|
||||
.downcast_ref::<String>()
|
||||
.cloned()
|
||||
.or_else(|| payload.downcast_ref::<&str>().map(|message| (*message).to_owned()))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_split_directory_reads_only_the_required_file() {
|
||||
let root = tempfile::tempdir().expect("tempdir");
|
||||
write(root.path(), "src/app/usecase.rs", "guarded();\n");
|
||||
|
||||
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
|
||||
|
||||
assert_eq!(set.describe(), "src/app/usecase.rs");
|
||||
assert!(set.contains("guarded();"));
|
||||
assert_eq!(set.count("guarded();"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_directory_files_follow_the_required_file_in_path_order() {
|
||||
let root = tempfile::tempdir().expect("tempdir");
|
||||
write(root.path(), "src/app/usecase.rs", "guarded();\n");
|
||||
write(root.path(), "src/app/usecase/zeta.rs", "guarded();\n");
|
||||
write(root.path(), "src/app/usecase/alpha.rs", "guarded();\nguarded();\n");
|
||||
write(root.path(), "src/app/usecase/tests/nested.rs", "guarded();\n");
|
||||
write(root.path(), "src/app/usecase/notes.txt", "guarded();\n");
|
||||
|
||||
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
|
||||
|
||||
assert_eq!(
|
||||
set.describe(),
|
||||
"src/app/usecase.rs, src/app/usecase/alpha.rs, src/app/usecase/tests/nested.rs, src/app/usecase/zeta.rs"
|
||||
);
|
||||
assert_eq!(set.count("guarded();"), 5, "non-.rs files are not source");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_required_file_fails_even_when_the_split_directory_exists() {
|
||||
let root = tempfile::tempdir().expect("tempdir");
|
||||
write(root.path(), "src/app/usecase/moved.rs", "guarded();\n");
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::read_under(&root_path, "src/app/usecase.rs", "src/app/usecase");
|
||||
});
|
||||
|
||||
assert!(message.contains("src/app/usecase.rs"), "panic must name the missing file: {message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_path_that_is_a_file_fails() {
|
||||
let root = tempfile::tempdir().expect("tempdir");
|
||||
write(root.path(), "src/app/usecase.rs", "guarded();\n");
|
||||
write(root.path(), "src/app/usecase", "not a directory\n");
|
||||
let root_path = root.path().to_path_buf();
|
||||
|
||||
let message = panic_message(move || {
|
||||
SourceSet::read_under(&root_path, "src/app/usecase.rs", "src/app/usecase");
|
||||
});
|
||||
|
||||
assert!(message.contains("must be a directory"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_does_not_join_a_needle_across_two_files() {
|
||||
let root = tempfile::tempdir().expect("tempdir");
|
||||
write(root.path(), "src/app/usecase.rs", "guard");
|
||||
write(root.path(), "src/app/usecase/tail.rs", "ed();\n");
|
||||
|
||||
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
|
||||
|
||||
assert_eq!(set.count("guarded();"), 0);
|
||||
assert!(!set.contains("guarded();"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unique_marker_absent_from_the_set_fails() {
|
||||
let root = tempfile::tempdir().expect("tempdir");
|
||||
write(root.path(), "src/app/usecase.rs", "fn other() {}\n");
|
||||
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
|
||||
|
||||
let message = panic_message(move || {
|
||||
set.after_unique_marker("fn guarded");
|
||||
});
|
||||
|
||||
assert!(message.contains("exactly once"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unique_marker_in_two_files_fails() {
|
||||
let root = tempfile::tempdir().expect("tempdir");
|
||||
write(root.path(), "src/app/usecase.rs", "fn guarded() {}\n");
|
||||
write(root.path(), "src/app/usecase/copy.rs", "fn guarded() {}\n");
|
||||
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
|
||||
|
||||
let message = panic_message(move || {
|
||||
set.after_unique_marker("fn guarded");
|
||||
});
|
||||
|
||||
assert!(message.contains("src/app/usecase/copy.rs"), "panic must name the set: {message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unique_marker_twice_in_one_file_fails() {
|
||||
let root = tempfile::tempdir().expect("tempdir");
|
||||
write(root.path(), "src/app/usecase.rs", "fn guarded() {}\nfn guarded() {}\n");
|
||||
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
|
||||
|
||||
let message = panic_message(move || {
|
||||
set.after_unique_marker("fn guarded");
|
||||
});
|
||||
|
||||
assert!(message.contains("exactly once"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unique_marker_returns_the_rest_of_its_own_file_only() {
|
||||
let root = tempfile::tempdir().expect("tempdir");
|
||||
write(root.path(), "src/app/usecase.rs", "fn head() {}\n");
|
||||
write(root.path(), "src/app/usecase/moved.rs", "fn guarded() { body(); }\n");
|
||||
write(root.path(), "src/app/usecase/next.rs", "fn later() { other(); }\n");
|
||||
let set = SourceSet::read_under(root.path(), "src/app/usecase.rs", "src/app/usecase");
|
||||
|
||||
let rest = set.after_unique_marker("fn guarded");
|
||||
|
||||
assert_eq!(rest, "() { body(); }\n");
|
||||
}
|
||||
}
|
||||
@@ -89,6 +89,8 @@ pub mod diagnose;
|
||||
pub mod embedded;
|
||||
pub mod error;
|
||||
pub mod foreground_observation;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod guard_sources;
|
||||
pub mod init;
|
||||
pub mod inspect;
|
||||
pub(crate) mod kms_deletion_gate;
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The access phase of a gateway handler: the legacy access hooks in their owned form.
|
||||
//!
|
||||
//! Responsible for: [`check`], what the legacy access phase runs once per request before any
|
||||
//! operation hook. Per-operation hooks join it here as their operations are registered.
|
||||
//! NOT responsible for: the decisions themselves. Every function here calls the function the
|
||||
//! legacy access phase calls (`storage/access.rs`), so the two stacks cannot answer differently.
|
||||
//! Upstream: each `impl Handler`. Downstream: the storage layer's access functions.
|
||||
//!
|
||||
//! The caller is resolved here on every request, and again by the gateway's `Authorizer` before
|
||||
//! it: the pinned gateway hands a handler no policy snapshot, so one resolution cannot be carried
|
||||
//! from the authorization stages to the handler.
|
||||
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::storage_api::server::http::access::{admit_operation, resolve_req_info};
|
||||
use rustfs_s3_types::S3Result;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Runs the per-request access check for `operation` and installs the caller in `envelope`.
|
||||
///
|
||||
/// The caller is the credential IAM holds for the access key the gateway verified, checked
|
||||
/// against the presented session token; an anonymous request resolves to no caller. The signed
|
||||
/// upload-size capabilities are then refused on any operation but their own, and the license is
|
||||
/// checked.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// What the legacy access phase answers: the credential refusals of the key lookup,
|
||||
/// `InvalidRequest` for a misplaced capability query, `AccessDenied` for a refused license, and
|
||||
/// `InternalError` when IAM or the license cannot be read.
|
||||
pub(crate) async fn check(
|
||||
server_ctx: &Arc<ServerContextSlot>,
|
||||
envelope: &mut RequestEnvelope,
|
||||
operation: &'static str,
|
||||
) -> S3Result<()> {
|
||||
let credentials = envelope.credentials();
|
||||
let access_key = credentials.map(|credentials| credentials.access_key().to_owned());
|
||||
let session_token = credentials
|
||||
.and_then(|credentials| credentials.session_token())
|
||||
.unwrap_or_default()
|
||||
.to_owned();
|
||||
let request_context = envelope.request_context().cloned();
|
||||
let req_info = resolve_req_info(server_ctx, access_key.as_deref(), &session_token, request_context).await?;
|
||||
envelope.install_req_info(req_info);
|
||||
admit_operation(envelope.headers(), envelope.uri().query(), operation)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! `ListBuckets` on the gateway stack.
|
||||
//!
|
||||
//! Responsible for: converting one accepted request into the envelope and input the RustFS use
|
||||
//! case takes, and its answer back.
|
||||
//! NOT responsible for: deciding which buckets the caller sees. The legacy access hook for this
|
||||
//! operation is empty and so is this one: `DefaultBucketUsecase::execute_list_buckets` decides
|
||||
//! per caller (every bucket with `s3:ListAllMyBuckets`, otherwise the buckets the caller may
|
||||
//! list or locate, `AccessDenied` when there is none), for both stacks.
|
||||
//! Upstream: the gateway's dispatch. Downstream: `crate::app::bucket_usecase`.
|
||||
|
||||
use super::super::GatewayBackend;
|
||||
use super::super::access;
|
||||
use super::super::error::handler_error;
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use rustfs_gateway::{Handler, HandlerResult, Operation, Req, Resp, dto};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
|
||||
impl Handler<dto::ListBuckets> for GatewayBackend {
|
||||
async fn call(&self, request: Req<dto::ListBuckets>) -> HandlerResult<dto::ListBuckets> {
|
||||
record_s3_op(S3Operation::ListBuckets);
|
||||
let mut envelope = RequestEnvelope::from_gateway(request.context(), self.server_ctx()).map_err(handler_error)?;
|
||||
access::check(self.server_ctx(), &mut envelope, <dto::ListBuckets as Operation>::NAME)
|
||||
.await
|
||||
.map_err(handler_error)?;
|
||||
let usecase = DefaultBucketUsecase::with_context(self.server_ctx().app_context());
|
||||
let output = usecase
|
||||
.execute_list_buckets(envelope, request.into_input())
|
||||
.await
|
||||
.map_err(handler_error)?;
|
||||
Ok(Resp::new(output))
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Bucket-level operations of the gateway stack, one file each.
|
||||
|
||||
mod list_buckets;
|
||||
@@ -1,373 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! Builds a [`RequestEnvelope`] from a request the gateway accepted, authenticated and routed.
|
||||
//!
|
||||
//! Responsible for: `RequestEnvelope::from_gateway`, the gateway edge's counterpart of the
|
||||
//! legacy edge's `from_s3s` (`storage/s3_compat/envelope.rs`).
|
||||
//! NOT responsible for: the facts the access phase attaches afterwards (`ReqInfo` and what the
|
||||
//! operation hooks retain); `super::access` installs those.
|
||||
//! Upstream: the gateway's handler request context. Downstream: every gateway handler.
|
||||
//!
|
||||
//! Every fact is copied from where the gateway or a RustFS layer in front of it put it: the
|
||||
//! request line and the accepted header lines from the context, the caller from the verified
|
||||
//! principal, and the payloads the connection and request layers attached from the transport
|
||||
//! values the gateway carried through unchanged.
|
||||
|
||||
use crate::app::object::request_body::BodyReadControl;
|
||||
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead};
|
||||
use crate::auth::{
|
||||
AuthType, VerifiedPresignedRequest, VerifiedSigV4Request, get_request_auth_type_with_query, get_session_token,
|
||||
};
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::shared_types::{NativeTlsTransport, RemoteAddr};
|
||||
use crate::storage_api::server::http::request_context::RequestContext;
|
||||
use http::{Extensions, HeaderMap, Uri};
|
||||
use rustfs_gateway::{RequestContextView, TransportExtensions};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::sync::Arc;
|
||||
|
||||
impl RequestEnvelope {
|
||||
/// Copies every request fact of `context` the app layer reads, for the server `server_ctx`.
|
||||
///
|
||||
/// A payload the request does not carry leaves its field `None`. The caller's secret key is
|
||||
/// copied only when the gateway handed it to this operation; the signature markers are set
|
||||
/// only for a caller the gateway verified.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// `InternalError` when the accepted request target cannot be read back as a URI, or a
|
||||
/// handed-over secret key is not text. Neither is the caller's fault.
|
||||
pub(crate) fn from_gateway(context: &RequestContextView, server_ctx: &Arc<ServerContextSlot>) -> S3Result<Self> {
|
||||
let uri = request_uri(context)?;
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in context.headers().iter_raw() {
|
||||
headers.append(name.clone(), value.clone());
|
||||
}
|
||||
let credentials = match context.principal() {
|
||||
Some(principal) => {
|
||||
let secret_key = principal
|
||||
.secret_key_from_authenticator_lookup()
|
||||
.map(|secret| secret.expose_secret());
|
||||
Some(request_credentials(
|
||||
principal.access_key_id(),
|
||||
secret_key,
|
||||
get_session_token(&uri, &headers),
|
||||
)?)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let mut extensions = Extensions::new();
|
||||
let transport = context.transport_extensions();
|
||||
copy_payload::<Option<RemoteAddr>>(transport, &mut extensions);
|
||||
copy_payload::<ClientInfo>(transport, &mut extensions);
|
||||
copy_payload::<NativeTlsTransport>(transport, &mut extensions);
|
||||
copy_payload::<RequestContext>(transport, &mut extensions);
|
||||
copy_payload::<BodyReadControl>(transport, &mut extensions);
|
||||
extensions.insert(Arc::clone(server_ctx));
|
||||
// The legacy access phase marks a request whose SigV4 signature its edge verified; here
|
||||
// the gateway verified it, and only a verified caller has a principal.
|
||||
if credentials.is_some() {
|
||||
match get_request_auth_type_with_query(&headers, uri.query()) {
|
||||
AuthType::Presigned => {
|
||||
extensions.insert(VerifiedPresignedRequest);
|
||||
extensions.insert(VerifiedSigV4Request);
|
||||
}
|
||||
AuthType::Signed => {
|
||||
extensions.insert(VerifiedSigV4Request);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let scope = context.verified_scope();
|
||||
let head = RequestHead {
|
||||
method: context.method().clone(),
|
||||
uri,
|
||||
headers,
|
||||
credentials,
|
||||
region: scope.map(|scope| scope.region().to_owned()),
|
||||
service: scope.map(|scope| scope.service().to_owned()),
|
||||
};
|
||||
Ok(Self::from_parts(head, extensions))
|
||||
}
|
||||
}
|
||||
|
||||
/// The request target as it arrived: the raw path and query, with the scheme and authority when
|
||||
/// the transport carried them (an absolute-form target, or HTTP/2).
|
||||
fn request_uri(context: &RequestContextView) -> S3Result<Uri> {
|
||||
let path_and_query = match context.raw_query() {
|
||||
"" => context.raw_path().to_owned(),
|
||||
query => format!("{}?{query}", context.raw_path()),
|
||||
};
|
||||
let builder = match (context.target_scheme(), context.target_authority()) {
|
||||
(Some(scheme), Some(authority)) => Uri::builder().scheme(scheme).authority(authority),
|
||||
_ => Uri::builder(),
|
||||
};
|
||||
builder
|
||||
.path_and_query(path_and_query)
|
||||
.build()
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "the accepted request target is not a URI"))
|
||||
}
|
||||
|
||||
/// The caller's credentials as the app layer holds them. `secret_key` is what the gateway handed
|
||||
/// over, which is nothing unless the operation opted in.
|
||||
fn request_credentials(access_key: &str, secret_key: Option<&[u8]>, session_token: Option<&str>) -> S3Result<RequestCredentials> {
|
||||
let session_token = session_token.map(str::to_owned);
|
||||
let Some(secret_key) = secret_key else {
|
||||
return Ok(RequestCredentials::without_secret_key(access_key.to_owned(), session_token));
|
||||
};
|
||||
let secret_key = std::str::from_utf8(secret_key)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "the caller's secret key is not text"))?;
|
||||
// The gateway refuses a credential whose secret is empty, so a handed-over secret is never
|
||||
// empty. Should one arrive, the request is refused rather than served with a secret that
|
||||
// every stored secret would have to be compared against.
|
||||
RequestCredentials::new(access_key.to_owned(), secret_key.to_owned(), session_token)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "the caller's secret key is empty"))
|
||||
}
|
||||
|
||||
/// Copies the payload of type `T` a layer in front of the gateway attached, when one did.
|
||||
fn copy_payload<T: Clone + Send + Sync + 'static>(transport: &TransportExtensions, extensions: &mut Extensions) {
|
||||
if let Some(payload) = transport.get::<T>() {
|
||||
extensions.insert(payload.clone());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::server::gateway_handlers::error::handler_error;
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderValue, Method, Request, StatusCode};
|
||||
use http_body_util::{BodyExt, Empty};
|
||||
use rustfs_gateway::sig::{AmzDate, PayloadMode, SigService, SigV4Signer, SigningCredentials, SigningRequest, SigningScope};
|
||||
use rustfs_gateway::{
|
||||
Credentials, Handler, HandlerResult, Limits, RegionSet, Req, Resp, ServiceBuilder, SigV4Authenticator, StaticCredentials,
|
||||
WireRequest, allow_when, dto,
|
||||
};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Mutex;
|
||||
use std::time::Instant;
|
||||
|
||||
const PEER: ([u8; 4], u16) = ([10, 0, 0, 7], 41000);
|
||||
const CLIENT: [u8; 4] = [203, 0, 113, 9];
|
||||
const REQUEST_ID: &str = "envelope-request-id";
|
||||
const SIGNER_ACCESS_KEY: &str = "AKIAENVELOPESIGNER";
|
||||
const SIGNER_SECRET_KEY: &str = "envelope-signer-secret-key-0123456789";
|
||||
|
||||
/// Records the envelope the gateway edge builds for the request it is handed.
|
||||
struct Probe {
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
seen: Mutex<Option<RequestEnvelope>>,
|
||||
}
|
||||
|
||||
impl Handler<dto::ListBuckets> for Probe {
|
||||
async fn call(&self, request: Req<dto::ListBuckets>) -> HandlerResult<dto::ListBuckets> {
|
||||
let envelope = RequestEnvelope::from_gateway(request.context(), &self.server_ctx).map_err(handler_error)?;
|
||||
*self.seen.lock().expect("probe lock") = Some(envelope);
|
||||
Ok(Resp::new(dto::ListBucketsOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends `request` through a gateway whose only handler is the probe, and returns what the
|
||||
/// probe built. The test authorizer admits the one probed operation and nothing else.
|
||||
async fn envelope_of(request: Request<Empty<Bytes>>, server_ctx: &Arc<ServerContextSlot>) -> RequestEnvelope {
|
||||
let probe = Arc::new(Probe {
|
||||
server_ctx: Arc::clone(server_ctx),
|
||||
seen: Mutex::new(None),
|
||||
});
|
||||
let regions = RegionSet::new(["us-east-1"]).expect("a region");
|
||||
let service = ServiceBuilder::new()
|
||||
.rustfs_profile()
|
||||
.authenticator(SigV4Authenticator::new(Arc::new(signer_credentials()), regions).rustfs_profile())
|
||||
.authorizer(allow_when(|request| request.operation == "ListBuckets"))
|
||||
.register::<dto::ListBuckets, _>(Arc::clone(&probe))
|
||||
.build()
|
||||
.expect("the probe service assembles");
|
||||
|
||||
let response = service.call(request).await;
|
||||
let status = response.status();
|
||||
if status != StatusCode::OK {
|
||||
let body = response
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.map(|body| body.to_bytes())
|
||||
.unwrap_or_default();
|
||||
panic!("the probe must be reached: {status} {}", String::from_utf8_lossy(&body));
|
||||
}
|
||||
probe
|
||||
.seen
|
||||
.lock()
|
||||
.expect("probe lock")
|
||||
.take()
|
||||
.expect("the probe saw the request")
|
||||
}
|
||||
|
||||
/// The one caller the probe service knows.
|
||||
fn signer_credentials() -> StaticCredentials {
|
||||
StaticCredentials::new().with(Credentials::new(SIGNER_ACCESS_KEY, SIGNER_SECRET_KEY.as_bytes()).expect("a valid key"))
|
||||
}
|
||||
|
||||
/// A ListBuckets request header-signed now by the probe service's one caller.
|
||||
fn signed_list_buckets() -> Request<Empty<Bytes>> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(http::header::HOST, HeaderValue::from_static("localhost"));
|
||||
let probe = list_buckets_request("/").body(Bytes::new()).expect("static request");
|
||||
let accepted = WireRequest::accept(probe, &Limits::default()).expect("an acceptable host");
|
||||
let stamp = AmzDate::parse(&chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string()).expect("a stamp");
|
||||
let scope = SigningScope::new(stamp.day(), "us-east-1", SigService::S3).expect("a scope");
|
||||
let credentials = SigningCredentials::new(SIGNER_ACCESS_KEY, SIGNER_SECRET_KEY.as_bytes()).expect("credentials");
|
||||
let mut signer = SigV4Signer::new(credentials, scope);
|
||||
let method = Method::GET;
|
||||
let signing = SigningRequest::new(
|
||||
&method,
|
||||
"/",
|
||||
"",
|
||||
&headers,
|
||||
accepted.host().raw_for_signing(),
|
||||
PayloadMode::Unsigned,
|
||||
stamp,
|
||||
);
|
||||
let signed = signer.sign_headers(&signing).expect("a signable request");
|
||||
let mut builder = Request::builder().method(method).uri("/");
|
||||
for (name, value) in signed.headers() {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
builder.body(Empty::new()).expect("a request")
|
||||
}
|
||||
|
||||
fn list_buckets_request(target: &str) -> http::request::Builder {
|
||||
Request::builder().method(Method::GET).uri(target).header("host", "localhost")
|
||||
}
|
||||
|
||||
fn request_context() -> RequestContext {
|
||||
RequestContext {
|
||||
request_id: REQUEST_ID.to_owned(),
|
||||
x_amz_request_id: REQUEST_ID.to_owned(),
|
||||
trace_id: None,
|
||||
span_id: None,
|
||||
start_time: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn from_gateway_copies_every_fact_of_the_accepted_request() {
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
let request = list_buckets_request("/?prefix=probe")
|
||||
.header("x-envelope-probe", "probe")
|
||||
.extension(Some(RemoteAddr(SocketAddr::from(PEER))))
|
||||
.extension(ClientInfo::direct(SocketAddr::new(IpAddr::from(CLIENT), 443)))
|
||||
.extension(NativeTlsTransport(true))
|
||||
.extension(request_context())
|
||||
.extension(BodyReadControl::default())
|
||||
.body(Empty::new())
|
||||
.expect("static request");
|
||||
|
||||
let envelope = envelope_of(request, &server_ctx).await;
|
||||
|
||||
assert_eq!(envelope.method(), Method::GET);
|
||||
assert_eq!(envelope.uri().path(), "/");
|
||||
assert_eq!(envelope.uri().query(), Some("prefix=probe"));
|
||||
assert_eq!(envelope.headers().get("x-envelope-probe"), Some(&HeaderValue::from_static("probe")));
|
||||
assert_eq!(envelope.remote_addr(), Some(SocketAddr::from(PEER)));
|
||||
assert_eq!(envelope.client_info().map(|info| info.real_ip), Some(IpAddr::from(CLIENT)));
|
||||
assert_eq!(envelope.native_tls_transport(), Some(true));
|
||||
assert_eq!(envelope.request_context().map(|context| context.request_id.as_str()), Some(REQUEST_ID));
|
||||
assert!(envelope.body_read_control().is_some(), "body_read_control");
|
||||
assert!(
|
||||
envelope.server_context().is_some_and(|slot| Arc::ptr_eq(slot, &server_ctx)),
|
||||
"server_context"
|
||||
);
|
||||
assert!(envelope.residual_extensions().is_empty(), "every copied payload is a named field");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn from_gateway_leaves_absent_payloads_none_and_an_anonymous_caller_unverified() {
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
let request = list_buckets_request("/").body(Empty::new()).expect("static request");
|
||||
|
||||
let envelope = envelope_of(request, &server_ctx).await;
|
||||
|
||||
assert!(envelope.credentials().is_none(), "credentials");
|
||||
assert_eq!(envelope.region(), None, "region");
|
||||
assert_eq!(envelope.service(), None, "service");
|
||||
assert_eq!(envelope.remote_addr(), None, "remote_addr");
|
||||
assert!(envelope.client_info().is_none(), "client_info");
|
||||
assert_eq!(envelope.native_tls_transport(), None, "native_tls_transport");
|
||||
assert!(envelope.request_context().is_none(), "request_context");
|
||||
assert!(envelope.body_read_control().is_none(), "body_read_control");
|
||||
assert!(envelope.verified_sigv4().is_none(), "verified_sigv4");
|
||||
assert!(envelope.verified_presigned().is_none(), "verified_presigned");
|
||||
assert!(envelope.req_info().is_none(), "the access phase installs req_info, not the edge");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn from_gateway_treats_a_peerless_remote_addr_as_absent() {
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
let request = list_buckets_request("/")
|
||||
.extension(None::<RemoteAddr>)
|
||||
.body(Empty::new())
|
||||
.expect("static request");
|
||||
|
||||
let envelope = envelope_of(request, &server_ctx).await;
|
||||
|
||||
assert_eq!(envelope.remote_addr(), None);
|
||||
assert!(envelope.residual_extensions().is_empty(), "the peerless marker is consumed, not kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credentials_carry_the_secret_only_when_the_gateway_handed_it_over() {
|
||||
let kept = request_credentials("AKIAENVELOPE", None, Some("token")).expect("credentials");
|
||||
assert_eq!(kept.access_key(), "AKIAENVELOPE");
|
||||
assert!(kept.secret_key().is_none());
|
||||
assert_eq!(kept.session_token(), Some("token"));
|
||||
|
||||
let handed = request_credentials("AKIAENVELOPE", Some(b"envelope-secret"), None).expect("credentials");
|
||||
assert_eq!(handed.secret_key(), Some("envelope-secret"));
|
||||
assert_eq!(handed.session_token(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_handed_over_secret_is_refused_instead_of_kept() {
|
||||
let error = request_credentials("AKIAENVELOPE", Some(b""), None).expect_err("empty");
|
||||
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
/// ListBuckets does not opt in to the caller's secret, so the gateway keeps it: the handler
|
||||
/// sees who signed and no secret, and a reader that needs one is refused instead of being
|
||||
/// handed an empty string to compare with.
|
||||
#[tokio::test]
|
||||
async fn an_operation_that_did_not_opt_in_reaches_the_handler_without_the_caller_s_secret() {
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
|
||||
let envelope = envelope_of(signed_list_buckets(), &server_ctx).await;
|
||||
|
||||
let credentials = envelope.credentials().expect("a signed request has a caller");
|
||||
assert_eq!(credentials.access_key(), SIGNER_ACCESS_KEY);
|
||||
assert!(credentials.secret_key().is_none(), "the secret stays with the gateway");
|
||||
assert!(credentials.require_secret_key().is_err(), "a reader of the secret fails closed");
|
||||
assert!(!credentials.secret_key_matches(""), "an absent secret equals no stored secret");
|
||||
assert!(!credentials.secret_key_matches(SIGNER_SECRET_KEY));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_secret_that_is_not_text_is_refused_instead_of_dropped() {
|
||||
let error = request_credentials("AKIAENVELOPE", Some(&[0xff, 0xfe]), None).expect_err("not text");
|
||||
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! The one conversion of a RustFS error into the gateway's handler error.
|
||||
//!
|
||||
//! Responsible for: [`handler_error`], called by every gateway handler and by nothing else.
|
||||
//! NOT responsible for: choosing an error (the use case's), or writing it (the gateway renders the
|
||||
//! document, the status and the request identifiers).
|
||||
//! Upstream: `rustfs_s3_types::S3Error`. Downstream: `rustfs_gateway::HandlerError`.
|
||||
//!
|
||||
//! The gateway's error carries a code with its status, a message, and response headers from a
|
||||
//! closed set. An error this function cannot carry whole is answered `500 InternalError` rather
|
||||
//! than sent with a member missing: a `416` without its `Content-Range`, or a throttle without
|
||||
//! its `Retry-After`, tells a client something RustFS did not say. Each operation slice that
|
||||
//! raises a header not listed in [`error_header`] adds it there, with its test.
|
||||
|
||||
use http::{HeaderName, HeaderValue, StatusCode};
|
||||
use rustfs_gateway::{ErrorHeader, HandlerError};
|
||||
use rustfs_gateway_types::ErrorCode;
|
||||
use rustfs_s3_types::S3Error;
|
||||
use std::borrow::Cow;
|
||||
|
||||
/// Answered when an error names a code without a status: the raiser of a custom code must set one.
|
||||
const MISSING_STATUS: &str = "the error carries a code without a status";
|
||||
/// Answered when an error carries a response header this edge cannot write.
|
||||
const UNWRITABLE_HEADER: &str = "the error carries a response header the gateway cannot write";
|
||||
|
||||
/// Converts a use-case error into the error the gateway answers with: the same code, status and
|
||||
/// message, and every response header it carries.
|
||||
pub(crate) fn handler_error(error: S3Error) -> HandlerError {
|
||||
let parts = error.into_parts();
|
||||
let Some(status) = parts.status.or_else(|| parts.code.status_code()) else {
|
||||
return HandlerError::internal_error(MISSING_STATUS);
|
||||
};
|
||||
let mut mapped = HandlerError::new(error_code(parts.code.as_str(), status), parts.message.unwrap_or(Cow::Borrowed("")));
|
||||
for (name, value) in parts.headers.iter().flatten() {
|
||||
let Some(header) = error_header(name, value) else {
|
||||
return HandlerError::internal_error(UNWRITABLE_HEADER);
|
||||
};
|
||||
mapped = mapped.with_header(header);
|
||||
}
|
||||
mapped
|
||||
}
|
||||
|
||||
/// The gateway's code for `name` answered with `status`: its own declared code when both agree,
|
||||
/// otherwise a code with exactly the spelling and status RustFS chose.
|
||||
fn error_code(name: &str, status: StatusCode) -> ErrorCode {
|
||||
match ErrorCode::known(name) {
|
||||
Some(declared) if declared.default_status() == status => declared,
|
||||
_ => ErrorCode::custom(name.to_owned(), status),
|
||||
}
|
||||
}
|
||||
|
||||
/// The gateway's spelling of one error response header, or `None` for a header it cannot write.
|
||||
fn error_header(name: &HeaderName, value: &HeaderValue) -> Option<ErrorHeader> {
|
||||
if name == http::header::RETRY_AFTER {
|
||||
let seconds = value.to_str().ok()?.parse().ok()?;
|
||||
return Some(ErrorHeader::RetryAfter { seconds });
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::HeaderMap;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
|
||||
#[test]
|
||||
fn a_declared_code_keeps_its_spelling_status_and_message() {
|
||||
let mapped = handler_error(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "AccessDenied");
|
||||
assert_eq!(mapped.code().default_status(), StatusCode::FORBIDDEN);
|
||||
assert_eq!(mapped.message(), "Access Denied");
|
||||
assert!(mapped.headers().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_internal_error_stays_a_server_fault() {
|
||||
let mapped = handler_error(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "InternalError");
|
||||
assert_eq!(mapped.code().default_status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert_eq!(mapped.message(), "Not init");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_explicit_status_overrides_the_declared_one() {
|
||||
let mut error = S3Error::with_message(S3ErrorCode::AccessDenied, "locked");
|
||||
error.set_status_code(StatusCode::CONFLICT);
|
||||
|
||||
let mapped = handler_error(error);
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "AccessDenied");
|
||||
assert_eq!(mapped.code().default_status(), StatusCode::CONFLICT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_custom_code_keeps_the_status_its_raiser_set() {
|
||||
let mut error = S3Error::with_message(S3ErrorCode::Custom("RustFsOnly".into()), "custom");
|
||||
error.set_status_code(StatusCode::SERVICE_UNAVAILABLE);
|
||||
|
||||
let mapped = handler_error(error);
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "RustFsOnly");
|
||||
assert_eq!(mapped.code().default_status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(mapped.message(), "custom");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_custom_code_without_a_status_is_a_server_fault_not_a_guess() {
|
||||
let mapped = handler_error(S3Error::with_message(S3ErrorCode::Custom("RustFsOnly".into()), "custom"));
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "InternalError");
|
||||
assert_eq!(mapped.code().default_status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert_eq!(mapped.message(), MISSING_STATUS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_message_is_not_invented() {
|
||||
let mapped = handler_error(S3Error::new(S3ErrorCode::AccessDenied));
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "AccessDenied");
|
||||
assert_eq!(mapped.message(), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_retry_after_header_is_carried() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(http::header::RETRY_AFTER, HeaderValue::from_static("7"));
|
||||
let mut error = S3Error::with_message(S3ErrorCode::SlowDown, "busy");
|
||||
error.set_headers(headers);
|
||||
|
||||
let mapped = handler_error(error);
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "SlowDown");
|
||||
assert_eq!(mapped.headers(), [ErrorHeader::RetryAfter { seconds: 7 }]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_header_the_gateway_cannot_write_fails_the_whole_error() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-rustfs-unlisted", HeaderValue::from_static("value"));
|
||||
let mut error = S3Error::with_message(S3ErrorCode::AccessDenied, "denied");
|
||||
error.set_headers(headers);
|
||||
|
||||
let mapped = handler_error(error);
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "InternalError");
|
||||
assert_eq!(mapped.message(), UNWRITABLE_HEADER);
|
||||
assert!(mapped.headers().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreadable_retry_after_fails_the_whole_error() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(http::header::RETRY_AFTER, HeaderValue::from_static("soon"));
|
||||
let mut error = S3Error::with_message(S3ErrorCode::SlowDown, "busy");
|
||||
error.set_headers(headers);
|
||||
|
||||
assert_eq!(handler_error(error).message(), UNWRITABLE_HEADER);
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The S3 operations the gateway stack serves, one file per operation (rustfs/backlog#2734).
|
||||
//!
|
||||
//! Responsible for: [`GatewayBackend`], the one backend every operation is implemented on, and
|
||||
//! [`register`], the explicit list of what is registered. An operation that is not in that list
|
||||
//! has no handler, and the gateway answers it `501 NotImplemented`.
|
||||
//! NOT responsible for: assembling the service (`server/gateway_stack.rs`), authenticating or
|
||||
//! routing a request (the gateway), or any S3 behaviour: a handler converts and delegates, and
|
||||
//! the RustFS use case it calls is the one the legacy stack calls.
|
||||
//! Upstream: the gateway's dispatch. Downstream: `crate::app` use cases.
|
||||
//!
|
||||
//! # The shape every handler has
|
||||
//!
|
||||
//! `<family>/<snake_op>.rs` holds exactly one `impl Handler<Op> for GatewayBackend`, in four
|
||||
//! steps and nothing else:
|
||||
//!
|
||||
//! 1. build the [`RequestEnvelope`](crate::app::RequestEnvelope) with `from_gateway`;
|
||||
//! 2. run the access phase: [`access::check`], then the operation's own access hook, which is
|
||||
//! the authoritative authorization and runs exactly where the legacy stack runs it;
|
||||
//! 3. call the use case with the envelope and the gateway's own input type;
|
||||
//! 4. map a refusal with [`error::handler_error`], the only place an error is converted.
|
||||
//!
|
||||
//! The gateway's `Authorizer` (`server/gateway_stack/authorizer.rs`) is an earlier gate in front
|
||||
//! of step 2 and never a replacement for it.
|
||||
|
||||
pub(crate) mod access;
|
||||
mod bucket;
|
||||
mod envelope;
|
||||
pub(crate) mod error;
|
||||
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use rustfs_gateway::{ServiceBuilder, dto};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The backend every gateway operation is implemented on: the server whose use cases answer.
|
||||
pub(crate) struct GatewayBackend {
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
}
|
||||
|
||||
impl GatewayBackend {
|
||||
pub(crate) fn new(server_ctx: Arc<ServerContextSlot>) -> Self {
|
||||
Self { server_ctx }
|
||||
}
|
||||
|
||||
pub(crate) fn server_ctx(&self) -> &Arc<ServerContextSlot> {
|
||||
&self.server_ctx
|
||||
}
|
||||
}
|
||||
|
||||
/// Registers every operation the gateway stack serves. Each slice adds one line here and one row
|
||||
/// to the authorization table; the census in `server/gateway_stack/authorizer.rs` holds the two
|
||||
/// lists equal.
|
||||
pub(crate) fn register(builder: ServiceBuilder, backend: &Arc<GatewayBackend>) -> ServiceBuilder {
|
||||
builder.register::<dto::ListBuckets, _>(Arc::clone(backend))
|
||||
}
|
||||
@@ -1,315 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! The gateway S3 stack of one server: its assembly, and the facts the host hands every request
|
||||
//! (rustfs/backlog#2734 T2.5).
|
||||
//!
|
||||
//! Responsible for: [`GatewayStack::assemble`], the one place the `rustfs-gateway` service is
|
||||
//! built for RustFS — the RustFS profile, signature verification against RustFS IAM, the RustFS
|
||||
//! authorizer, explicit deadlines, and the registered handlers — and [`GatewayStack::call`],
|
||||
//! which attaches the host's facts to a request before the gateway accepts it.
|
||||
//! NOT responsible for: choosing the stack (`server/stack.rs`), the middleware around it
|
||||
//! (`server/http.rs`), or any S3 behaviour (`server/gateway_handlers` and the use cases).
|
||||
//! Upstream: `server/http.rs`. Downstream: `rustfs-gateway`, `rustfs-gateway-adapter`.
|
||||
//!
|
||||
//! # What the host keeps
|
||||
//!
|
||||
//! RustFS's own layers stay in front of the gateway and the gateway's counterparts stay off
|
||||
//! (rustfs/backlog#2771): CORS is answered by `ConditionalCorsLayer`, so no CORS source is
|
||||
//! installed and the gateway writes no CORS header; the SSE-C transport layer runs first and the
|
||||
//! gateway's own strict gate stays behind it; and the request identifier is the one
|
||||
//! `ExternalRequestContextLayer` assigned, handed over as the gateway's `HostRequestId`, so an
|
||||
//! answer and RustFS's logs name a request identically.
|
||||
|
||||
// The stack is under construction, so a build without debug assertions must not carry it. The
|
||||
// message is checked by `scripts/check_gateway_stack_release_boundary.sh`.
|
||||
#[cfg(not(debug_assertions))]
|
||||
compile_error!("gateway-stack is restricted to debug builds");
|
||||
|
||||
mod authorizer;
|
||||
|
||||
use self::authorizer::{AUTHORIZATION_TABLE, OwnedDecisionCore, RequestLine, RustfsAuthorizer};
|
||||
use crate::runtime_sources::{ServerContextSlot, current_action_credentials};
|
||||
use crate::server::gateway_handlers::{GatewayBackend, register};
|
||||
use crate::server::ssec_transport::is_secure_transport;
|
||||
use crate::shared_types::{NativeTlsTransport, RemoteAddr};
|
||||
use crate::storage_api::server::http::request_context::RequestContext;
|
||||
use http::{Extensions, Request, Response};
|
||||
use rustfs_config::RUSTFS_REGION;
|
||||
use rustfs_gateway::{
|
||||
AssemblyError, Authorizer, BoxFuture, ClientAddr, DEFAULT_MAX_BUFFERED_BODY_BYTES, HandlerDeadlineClass,
|
||||
HandlerDeadlineConfig, HostRequestId, LegacyRustfsVirtualHosts, MintedTraces, RegionSet, RequestBodyDeadlineConfig,
|
||||
ServiceBuilder, ServiceConfig, SigV4Authenticator, SseConfig, TransportSecurity,
|
||||
};
|
||||
use rustfs_gateway_adapter::{IamAdapters, IamKeyLookup, IamPrincipalLookup, RootCredentials, RootCredentialsError, RootSource};
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::manager::get_token_signing_key;
|
||||
use rustfs_iam::store::object::ObjectStore;
|
||||
use rustfs_iam::sys::IamSys;
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
/// What the assembly takes from the server it is built for.
|
||||
pub(crate) struct GatewayAssembly {
|
||||
pub(crate) server_ctx: Arc<ServerContextSlot>,
|
||||
/// The configured root credentials, checked once here. Requests read the live ones.
|
||||
pub(crate) root_access_key: String,
|
||||
pub(crate) root_secret_key: String,
|
||||
/// The configured region; the default region signs when none is configured.
|
||||
pub(crate) region: Option<String>,
|
||||
/// The virtual-hosted-style base domains of the S3 listener, as the legacy stack reads them.
|
||||
pub(crate) server_domains: Vec<String>,
|
||||
/// `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT`: the longest silence between two body reads. Zero
|
||||
/// disables the bound, as it does on the legacy stack.
|
||||
pub(crate) body_read_timeout: Duration,
|
||||
/// Whether Keystone authentication is enabled in configuration (`RUSTFS_KEYSTONE_ENABLE`).
|
||||
pub(crate) keystone_enabled: bool,
|
||||
}
|
||||
|
||||
/// Whether Keystone authentication is switched on in configuration. A Keystone configuration
|
||||
/// that cannot be read counts as switched on: only a readable configuration can say it is off.
|
||||
pub(crate) fn keystone_enabled_in_configuration() -> bool {
|
||||
rustfs_keystone::KeystoneConfig::from_env().map_or(true, |config| config.enable)
|
||||
}
|
||||
|
||||
/// Why the gateway stack could not be assembled. Startup fails with it; the legacy stack is never
|
||||
/// started in its place.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum GatewayAssemblyError {
|
||||
#[error("the root credentials cannot verify signatures on the gateway stack: {0}")]
|
||||
RootCredentials(RootCredentialsError),
|
||||
#[error("region {region:?} cannot be a signing region on the gateway stack")]
|
||||
Region { region: String },
|
||||
#[error("the server domains cannot be served by the gateway stack: {0:?}")]
|
||||
ServerDomains(rustfs_gateway::LegacyDomainError),
|
||||
#[error("the gateway service refused its assembly: {0:?}")]
|
||||
Service(AssemblyError),
|
||||
#[error(
|
||||
"RUSTFS_S3_STACK=gateway cannot be served while RUSTFS_KEYSTONE_ENABLE is on: the gateway admits a request that \
|
||||
carries only X-Auth-Token as anonymous while RustFS reads the Keystone identity behind it; Keystone support on the \
|
||||
gateway stack is rustfs/gateway#1586"
|
||||
)]
|
||||
KeystoneEnabled,
|
||||
}
|
||||
|
||||
/// The assembled gateway service.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct GatewayStack {
|
||||
service: rustfs_gateway::S3Service,
|
||||
}
|
||||
|
||||
impl GatewayStack {
|
||||
/// Assembles the gateway stack for one server.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`GatewayAssemblyError`] naming the input the gateway refused. The secret key is never
|
||||
/// part of it.
|
||||
pub(crate) fn assemble(assembly: GatewayAssembly) -> Result<Self, GatewayAssemblyError> {
|
||||
let iam = Arc::new(ServerIam {
|
||||
server_ctx: Arc::clone(&assembly.server_ctx),
|
||||
});
|
||||
let root = Arc::new(live_root(Arc::clone(&assembly.server_ctx)));
|
||||
let core = OwnedDecisionCore::new(Arc::clone(&assembly.server_ctx));
|
||||
let authorizer = RustfsAuthorizer::new(AUTHORIZATION_TABLE, Arc::new(core));
|
||||
Self::assemble_with(assembly, iam, root, authorizer)
|
||||
}
|
||||
|
||||
fn assemble_with(
|
||||
assembly: GatewayAssembly,
|
||||
iam: Arc<dyn IamPrincipalLookup>,
|
||||
root: Arc<dyn RootSource>,
|
||||
authorizer: impl Authorizer,
|
||||
) -> Result<Self, GatewayAssemblyError> {
|
||||
// The pinned gateway admits a request that carries only `X-Auth-Token` as anonymous and
|
||||
// runs no authenticator for it, while the RustFS code behind the gateway reads the
|
||||
// Keystone identity: the request would be authorized as one caller and served as
|
||||
// another. Refused until the gateway authenticates the token (rustfs/gateway#1586).
|
||||
if assembly.keystone_enabled {
|
||||
return Err(GatewayAssemblyError::KeystoneEnabled);
|
||||
}
|
||||
// A root the gateway could not verify a signature with is a startup error, never a
|
||||
// server that silently has no root. Requests read the live root from `root`.
|
||||
RootCredentials::new(&assembly.root_access_key, &assembly.root_secret_key, &assembly.root_secret_key)
|
||||
.map_err(GatewayAssemblyError::RootCredentials)?;
|
||||
let region = assembly.region.as_deref().unwrap_or(RUSTFS_REGION);
|
||||
let regions = RegionSet::new([region]).map_err(|_| GatewayAssemblyError::Region {
|
||||
region: region.to_owned(),
|
||||
})?;
|
||||
let hosts = LegacyRustfsVirtualHosts::new(&assembly.server_domains).map_err(GatewayAssemblyError::ServerDomains)?;
|
||||
let backend = Arc::new(GatewayBackend::new(Arc::clone(&assembly.server_ctx)));
|
||||
|
||||
// Every legacy RustFS reading of a request, a credential scope and an answer, in the
|
||||
// gateway's one reviewable preset; its other half is on the authenticator below.
|
||||
let builder = ServiceBuilder::new().rustfs_profile();
|
||||
// The credential provider and the policy source go in together, over one IAM and one
|
||||
// live root, so a key cannot be root to one and an ordinary user to the other.
|
||||
let builder = IamAdapters::new(iam, root)
|
||||
.install(builder, |credentials| SigV4Authenticator::new(credentials, regions).rustfs_profile());
|
||||
let builder = builder
|
||||
.authorizer(authorizer)
|
||||
// The outer SSE-C transport layer answers first; the gateway's own gate stays strict
|
||||
// behind it.
|
||||
.sse_config(SseConfig::strict())
|
||||
.host_resolver(hosts)
|
||||
// The fallback only: `attach_host_facts` hands RustFS's own identifier over.
|
||||
.trace_source(MintedTraces::with_uuid_request_ids());
|
||||
let (builder, _settings) = builder.config(service_config(assembly.body_read_timeout));
|
||||
let service = register(builder, &backend).build().map_err(GatewayAssemblyError::Service)?;
|
||||
Ok(Self { service })
|
||||
}
|
||||
|
||||
/// The operations the assembled service answers, read from its registry. Everything else is
|
||||
/// answered `501 NotImplemented`.
|
||||
pub(crate) fn registered_operations(&self) -> Vec<&'static str> {
|
||||
self.service.operations().collect()
|
||||
}
|
||||
|
||||
/// Answers one request on the gateway stack, with the host's facts attached.
|
||||
pub(crate) fn call<B>(&self, mut request: Request<B>) -> impl Future<Output = Response<rustfs_gateway::Body>> + Send + 'static
|
||||
where
|
||||
B: http_body::Body + Send + 'static,
|
||||
B::Data: Send,
|
||||
B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
{
|
||||
// The authorizer is handed the headers and the query but not the request line, and the
|
||||
// shared decision core is asked about the request as it arrived.
|
||||
let line = RequestLine {
|
||||
method: request.method().clone(),
|
||||
uri: request.uri().clone(),
|
||||
};
|
||||
request.extensions_mut().insert(line);
|
||||
attach_host_facts(request.extensions_mut());
|
||||
let mut service = self.service.clone();
|
||||
let answer = tower::Service::call(&mut service, request);
|
||||
async move {
|
||||
match answer.await {
|
||||
Ok(response) => response,
|
||||
Err(never) => match never {},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The gateway's request settings for RustFS, every deadline stated instead of inherited.
|
||||
///
|
||||
/// No handler runs under a total deadline, object data included: the legacy stack has none, and a
|
||||
/// large transfer must not be cut off for its duration. A committed answer is not bounded either.
|
||||
/// The request body keeps RustFS's own idle bound and no throughput floor.
|
||||
fn service_config(body_read_timeout: Duration) -> ServiceConfig {
|
||||
let handlers = HandlerDeadlineConfig::default()
|
||||
.without_deadline(HandlerDeadlineClass::Standard)
|
||||
.without_deadline(HandlerDeadlineClass::Extended)
|
||||
.without_commit_progress_deadline();
|
||||
ServiceConfig::new(DEFAULT_MAX_BUFFERED_BODY_BYTES)
|
||||
.with_handler_deadlines(handlers)
|
||||
.with_request_body_deadlines(body_deadlines(body_read_timeout))
|
||||
}
|
||||
|
||||
/// RustFS's body read timeout as the gateway's idle deadlines: the same silence is allowed before
|
||||
/// the first byte and between reads, and zero lifts both.
|
||||
fn body_deadlines(read_timeout: Duration) -> RequestBodyDeadlineConfig {
|
||||
RequestBodyDeadlineConfig::new(read_timeout, read_timeout)
|
||||
.unwrap_or(RequestBodyDeadlineConfig::S3.without_idle_deadlines())
|
||||
.without_throughput_floor()
|
||||
}
|
||||
|
||||
/// Attaches what the gateway reads about a request from its host: whether the transport was
|
||||
/// secure, the client address after RustFS's trusted-proxy correction, and RustFS's request
|
||||
/// identifier. Each is derived from what RustFS's own layers already attached; no header is read.
|
||||
fn attach_host_facts(extensions: &mut Extensions) {
|
||||
let client_info = extensions.get::<ClientInfo>();
|
||||
let native_tls = extensions.get::<NativeTlsTransport>().is_some_and(|transport| transport.0);
|
||||
let security = if is_secure_transport(native_tls, client_info) {
|
||||
TransportSecurity::Encrypted
|
||||
} else {
|
||||
TransportSecurity::Plaintext
|
||||
};
|
||||
let client_ip = client_info.map(|info| info.real_ip).or_else(|| {
|
||||
extensions
|
||||
.get::<Option<RemoteAddr>>()
|
||||
.and_then(|peer| peer.as_ref().map(|peer| peer.0.ip()))
|
||||
});
|
||||
// An identifier outside the gateway's alphabet is not handed over; the gateway mints its own.
|
||||
let request_id = extensions
|
||||
.get::<RequestContext>()
|
||||
.and_then(|context| HostRequestId::new(&context.request_id).ok());
|
||||
|
||||
extensions.insert(security);
|
||||
if let Some(client_ip) = client_ip {
|
||||
extensions.insert(ClientAddr::from_peer(client_ip));
|
||||
}
|
||||
if let Some(request_id) = request_id {
|
||||
extensions.insert(request_id);
|
||||
}
|
||||
}
|
||||
|
||||
/// The root credentials of one server as they are now, read on every call from where the legacy
|
||||
/// stack reads them, so a root change reaches both stacks at once. `None` while they are not
|
||||
/// initialized or cannot verify a signature, which the adapters answer as unavailable.
|
||||
fn live_root(server_ctx: Arc<ServerContextSlot>) -> impl Fn() -> Option<Arc<RootCredentials>> + Send + Sync + 'static {
|
||||
move || {
|
||||
let root = match server_ctx.app_context() {
|
||||
Some(context) => context.action_credentials().get(),
|
||||
None => current_action_credentials(),
|
||||
}?;
|
||||
// IAM signs STS session tokens with its token-signing key, which is the root secret
|
||||
// until IAM has published one of its own.
|
||||
let signing_key = get_token_signing_key().unwrap_or_else(|| root.secret_key.clone());
|
||||
RootCredentials::new(&root.access_key, &root.secret_key, &signing_key)
|
||||
.ok()
|
||||
.map(Arc::new)
|
||||
}
|
||||
}
|
||||
|
||||
/// The IAM system of one server, looked up when a credential is: IAM is installed after the
|
||||
/// listener starts, so the assembly cannot hold a handle.
|
||||
struct ServerIam {
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
}
|
||||
|
||||
impl ServerIam {
|
||||
fn ready(&self) -> Option<Arc<IamSys<ObjectStore>>> {
|
||||
self.server_ctx
|
||||
.app_context()
|
||||
.filter(|context| context.iam().is_ready())
|
||||
.map(|context| context.iam().handle())
|
||||
}
|
||||
}
|
||||
|
||||
impl IamKeyLookup for ServerIam {
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready().is_some()
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
|
||||
Box::pin(async move {
|
||||
let Some(iam) = self.ready() else {
|
||||
return Err(IamError::IamSysNotInitialized);
|
||||
};
|
||||
iam.check_key(access_key).await
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl IamPrincipalLookup for ServerIam {
|
||||
fn group_memberships(&self, user: &str) -> Vec<String> {
|
||||
self.ready().map(|iam| iam.user_group_memberships(user)).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -1,687 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! The gateway `Authorizer` of the RustFS assembly: an early gate, never the authorization.
|
||||
//!
|
||||
//! Responsible for: [`AUTHORIZATION_TABLE`], one row per registered operation, and
|
||||
//! [`RustfsAuthorizer`], which answers both gateway authorization stages from it.
|
||||
//! NOT responsible for: the authoritative decision. That is the legacy access hook in its owned
|
||||
//! form, run by the operation's handler immediately before its use case
|
||||
//! (`server/gateway_handlers`), where the legacy stack runs it; decisions, error codes and their
|
||||
//! order are therefore the same code on both stacks (rustfs/backlog#2761).
|
||||
//! Upstream: the gateway's route and input authorization stages. Downstream: the shared
|
||||
//! authorization core, through [`DecisionCore`].
|
||||
//!
|
||||
//! # What this gate may do
|
||||
//!
|
||||
//! It may refuse a request the hook would certainly refuse too, and nothing else. It is never
|
||||
//! the only check, and it never allows by default:
|
||||
//!
|
||||
//! - an operation without a row is denied;
|
||||
//! - a `RouteDecidable` row asks the shared decision core the hook asks, with the same facts;
|
||||
//! - a `DeferToHandlerHook` row evaluates only its [`RouteGate`], which names exactly what the
|
||||
//! route stage can know, and leaves the rest to the hook;
|
||||
//! - a `LegacyNoCheck` row is an operation whose legacy hook is empty, said so explicitly;
|
||||
//! - a core that cannot answer is `Indeterminate`, which the gateway refuses.
|
||||
//!
|
||||
//! Plain rows therefore evaluate policy twice, once here and once in the hook. That is accepted
|
||||
//! until it is measured.
|
||||
//!
|
||||
//! A row may be `RouteDecidable` only when its denial concerns the one routed resource and the
|
||||
//! one routed action. An operation whose denial can concern a second resource (a copy source) or
|
||||
//! a version-specific action (a versioned read) is a `DeferToHandlerHook` row: the hook writes
|
||||
//! the denial record with the resource and action it actually refused, and an early refusal
|
||||
//! here would record the wrong ones.
|
||||
//!
|
||||
//! # Deferred rows
|
||||
//!
|
||||
//! Every row that is not `RouteDecidable` is listed here, and the census below holds the table
|
||||
//! to these two lists, so a deferral cannot be added without this section changing.
|
||||
//!
|
||||
//! `DeferToHandlerHook`:
|
||||
//!
|
||||
//! - `ListBuckets`: the legacy hook is empty and the use case decides per caller. It lists every
|
||||
//! bucket for a caller allowed `s3:ListAllMyBuckets`, otherwise the buckets the caller may list
|
||||
//! or locate, and refuses when there is none; that needs the bucket inventory. The route stage
|
||||
//! refuses an anonymous caller, which the use case refuses with the same answer.
|
||||
//!
|
||||
//! `LegacyNoCheck`: none.
|
||||
|
||||
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead, authorize_owned_request};
|
||||
use crate::auth::get_session_token;
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::shared_types::RemoteAddr;
|
||||
use crate::storage_api::server::http::access::resolve_req_info;
|
||||
use crate::storage_api::server::http::request_context::RequestContext;
|
||||
use http::{Extensions, HeaderMap, Method, Uri};
|
||||
use rustfs_gateway::{
|
||||
Authorizer, AuthzRequest, BoxFuture, Decision, InputAuthzRequest, InputDecisions, RequestContext as GatewayRequestContext,
|
||||
};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The resource an operation's action is asked about.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ResourceForm {
|
||||
/// The service itself: no bucket and no key.
|
||||
Service,
|
||||
/// One bucket.
|
||||
#[cfg_attr(not(test), expect(dead_code, reason = "constructed by the first bucket-level row"))]
|
||||
Bucket,
|
||||
/// One object in one bucket.
|
||||
#[cfg_attr(not(test), expect(dead_code, reason = "constructed by the first object-level row"))]
|
||||
Object,
|
||||
}
|
||||
|
||||
/// What a deferred row's route stage evaluates: only facts that do not depend on the body.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum RouteGate {
|
||||
/// Refuses a request that presented no credentials. A request with a verified caller passes
|
||||
/// to the hook, which decides.
|
||||
AuthenticatedCaller,
|
||||
}
|
||||
|
||||
/// How much of an operation's decision the route stage can make.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum RouteStage {
|
||||
/// The route stage asks the shared decision core the same question, with the same facts, as
|
||||
/// the hook.
|
||||
#[cfg_attr(not(test), expect(dead_code, reason = "constructed by the first plain operation row"))]
|
||||
RouteDecidable,
|
||||
/// The route stage evaluates `gate` and the hook in the handler decides the rest.
|
||||
DeferToHandlerHook {
|
||||
/// Exactly what the route stage evaluates.
|
||||
gate: RouteGate,
|
||||
/// Why the route stage cannot decide the whole of it.
|
||||
reason: &'static str,
|
||||
},
|
||||
/// The legacy hook for this operation is empty, so there is nothing to evaluate early.
|
||||
#[cfg_attr(
|
||||
not(test),
|
||||
expect(dead_code, reason = "constructed by the row of an operation whose legacy hook is empty")
|
||||
)]
|
||||
LegacyNoCheck {
|
||||
/// Why legacy checks nothing.
|
||||
reason: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
/// The checks an operation's legacy hook performs beyond its one action, which the shared
|
||||
/// decision core does not perform. There is no default: every row states this column.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ExtraChecks {
|
||||
/// The hook asks its one action and nothing else.
|
||||
#[cfg_attr(not(test), expect(dead_code, reason = "constructed by the first plain operation row"))]
|
||||
None,
|
||||
/// The hook, or the use case it defers to, also performs these.
|
||||
Performed(&'static [&'static str]),
|
||||
}
|
||||
|
||||
/// One operation's authorization, as the legacy access hook performs it.
|
||||
///
|
||||
/// A struct literal names every column, so a row cannot leave one unspecified.
|
||||
#[cfg_attr(
|
||||
not(test),
|
||||
expect(
|
||||
dead_code,
|
||||
reason = "the body-input and extra-check columns record what the handler hook does and are read by the census"
|
||||
)
|
||||
)]
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) struct AuthorizationRow {
|
||||
/// The operation, by the name it is registered under.
|
||||
pub(crate) operation: &'static str,
|
||||
/// The action the legacy hook asks, mapped from the operation and not from the gateway's
|
||||
/// spelling of its action: the two differ for 22 operations, and legacy's is the baseline.
|
||||
pub(crate) action: Action,
|
||||
pub(crate) resource: ResourceForm,
|
||||
pub(crate) route_stage: RouteStage,
|
||||
/// The body-derived inputs the hook needs, which no route stage has.
|
||||
pub(crate) body_inputs: &'static [&'static str],
|
||||
pub(crate) extra_checks: ExtraChecks,
|
||||
}
|
||||
|
||||
/// Every operation the gateway stack registers. Kept equal to the registered set by the census
|
||||
/// in `server/gateway_stack.rs`.
|
||||
pub(crate) static AUTHORIZATION_TABLE: &[AuthorizationRow] = &[AuthorizationRow {
|
||||
operation: "ListBuckets",
|
||||
action: Action::S3Action(S3Action::ListAllMyBucketsAction),
|
||||
resource: ResourceForm::Service,
|
||||
route_stage: RouteStage::DeferToHandlerHook {
|
||||
gate: RouteGate::AuthenticatedCaller,
|
||||
reason: "the use case decides per caller from the bucket inventory",
|
||||
},
|
||||
body_inputs: &[],
|
||||
extra_checks: ExtraChecks::Performed(&[
|
||||
"a caller without s3:ListAllMyBuckets sees the buckets it may list (s3:ListBucket) or locate (s3:GetBucketLocation), \
|
||||
judged by IAM alone, and is refused when there is none",
|
||||
]),
|
||||
}];
|
||||
|
||||
/// The request line as it reached RustFS, attached by the host before the gateway accepts the
|
||||
/// request. The gateway hands an authorizer the headers and the query, but neither the method
|
||||
/// nor the target, and the shared core is asked about the request as it arrived.
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct RequestLine {
|
||||
pub(crate) method: Method,
|
||||
pub(crate) uri: Uri,
|
||||
}
|
||||
|
||||
/// The request facts the shared decision core is asked with, each copied from where the gateway
|
||||
/// or a RustFS layer in front of it put it.
|
||||
pub(crate) struct RouteFacts<'a> {
|
||||
/// The access key the gateway verified; `None` for an anonymous request.
|
||||
pub(crate) access_key: Option<&'a str>,
|
||||
/// The bucket, key and version the gateway routed, when the request names them.
|
||||
pub(crate) bucket: Option<&'a str>,
|
||||
pub(crate) key: Option<&'a str>,
|
||||
pub(crate) version_id: Option<&'a str>,
|
||||
/// The accepted header lines; `None` when the gateway handed over none.
|
||||
pub(crate) headers: Option<HeaderMap>,
|
||||
pub(crate) request_line: Option<&'a RequestLine>,
|
||||
pub(crate) client_info: Option<&'a ClientInfo>,
|
||||
pub(crate) remote_addr: Option<SocketAddr>,
|
||||
pub(crate) request_context: Option<&'a RequestContext>,
|
||||
}
|
||||
|
||||
/// The shared authorization decision core, as the route stage of a `RouteDecidable` row asks it.
|
||||
pub(crate) trait DecisionCore: Send + Sync + 'static {
|
||||
/// Decides `row`'s action for the caller and resource of `facts`: `Ok` to allow,
|
||||
/// `AccessDenied` to deny, and any other error when the core could not answer.
|
||||
fn decide<'a>(&'a self, row: &'a AuthorizationRow, facts: &'a RouteFacts<'a>) -> BoxFuture<'a, S3Result<()>>;
|
||||
}
|
||||
|
||||
/// The decision core both stacks share: the owned authorization entry point, asked about the
|
||||
/// request the route stage sees.
|
||||
///
|
||||
/// The caller is resolved by the function the legacy access check resolves it with, and the
|
||||
/// decision is `authorize_owned_request`, which the handler's access hook calls too. It is only
|
||||
/// the policy decision: a row that needs a body-derived input or a composite check is not
|
||||
/// `RouteDecidable`.
|
||||
pub(crate) struct OwnedDecisionCore {
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
}
|
||||
|
||||
impl OwnedDecisionCore {
|
||||
pub(crate) fn new(server_ctx: Arc<ServerContextSlot>) -> Self {
|
||||
Self { server_ctx }
|
||||
}
|
||||
|
||||
async fn authorize(&self, row: &AuthorizationRow, facts: &RouteFacts<'_>) -> S3Result<()> {
|
||||
let (Some(line), Some(headers)) = (facts.request_line, facts.headers.as_ref()) else {
|
||||
return Err(unanswerable("the route stage was handed no request line or headers"));
|
||||
};
|
||||
let session_token = get_session_token(&line.uri, headers).unwrap_or_default().to_owned();
|
||||
let mut req_info =
|
||||
resolve_req_info(&self.server_ctx, facts.access_key, &session_token, facts.request_context.cloned()).await?;
|
||||
match row.resource {
|
||||
ResourceForm::Service => {}
|
||||
ResourceForm::Bucket => req_info.bucket = Some(routed(facts.bucket)?),
|
||||
ResourceForm::Object => {
|
||||
req_info.bucket = Some(routed(facts.bucket)?);
|
||||
req_info.object = Some(routed(facts.key)?);
|
||||
req_info.version_id = facts.version_id.map(str::to_owned);
|
||||
}
|
||||
}
|
||||
|
||||
let credentials = facts.access_key.map(|access_key| {
|
||||
let session_token = (!session_token.is_empty()).then(|| session_token.clone());
|
||||
RequestCredentials::without_secret_key(access_key.to_owned(), session_token)
|
||||
});
|
||||
let mut extensions = Extensions::new();
|
||||
extensions.insert(facts.remote_addr.map(RemoteAddr));
|
||||
if let Some(client_info) = facts.client_info {
|
||||
extensions.insert(client_info.clone());
|
||||
}
|
||||
if let Some(request_context) = facts.request_context {
|
||||
extensions.insert(request_context.clone());
|
||||
}
|
||||
extensions.insert(Arc::clone(&self.server_ctx));
|
||||
let head = RequestHead {
|
||||
method: line.method.clone(),
|
||||
uri: line.uri.clone(),
|
||||
headers: headers.clone(),
|
||||
credentials,
|
||||
region: None,
|
||||
service: None,
|
||||
};
|
||||
let mut envelope = RequestEnvelope::from_parts(head, extensions);
|
||||
envelope.install_req_info(req_info);
|
||||
authorize_owned_request(&mut envelope, row.action, None).await
|
||||
}
|
||||
}
|
||||
|
||||
impl DecisionCore for OwnedDecisionCore {
|
||||
fn decide<'a>(&'a self, row: &'a AuthorizationRow, facts: &'a RouteFacts<'a>) -> BoxFuture<'a, S3Result<()>> {
|
||||
Box::pin(self.authorize(row, facts))
|
||||
}
|
||||
}
|
||||
|
||||
/// The routed name a row's resource form requires. Its absence is a question the core cannot
|
||||
/// answer, never a decision about some other resource.
|
||||
fn routed(name: Option<&str>) -> S3Result<String> {
|
||||
name.map(str::to_owned)
|
||||
.ok_or_else(|| unanswerable("the route stage was handed no name for the row's resource"))
|
||||
}
|
||||
|
||||
fn unanswerable(message: &'static str) -> S3Error {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, message)
|
||||
}
|
||||
|
||||
/// What the table decides about one request by itself, and what it leaves to the core.
|
||||
enum Question<'a> {
|
||||
Decided(Decision),
|
||||
AskCore(&'static AuthorizationRow, Box<RouteFacts<'a>>),
|
||||
}
|
||||
|
||||
/// The RustFS implementation of the gateway's two authorization stages.
|
||||
pub(crate) struct RustfsAuthorizer {
|
||||
table: &'static [AuthorizationRow],
|
||||
core: Arc<dyn DecisionCore>,
|
||||
}
|
||||
|
||||
impl RustfsAuthorizer {
|
||||
/// The authorizer over `table`, asking `core` for `RouteDecidable` rows.
|
||||
pub(crate) fn new(table: &'static [AuthorizationRow], core: Arc<dyn DecisionCore>) -> Self {
|
||||
Self { table, core }
|
||||
}
|
||||
|
||||
/// Reads `operation`'s row. `facts` is built only for a row that asks the core.
|
||||
fn question<'f>(&self, operation: &str, anonymous: bool, facts: impl FnOnce() -> RouteFacts<'f>) -> Question<'f> {
|
||||
let table: &'static [AuthorizationRow] = self.table;
|
||||
let Some(row) = table.iter().find(|row| row.operation == operation) else {
|
||||
// No row, no answer to give: refused, never allowed.
|
||||
return Question::Decided(Decision::Deny);
|
||||
};
|
||||
match row.route_stage {
|
||||
RouteStage::RouteDecidable => Question::AskCore(row, Box::new(facts())),
|
||||
RouteStage::DeferToHandlerHook { gate, .. } => Question::Decided(match gate {
|
||||
RouteGate::AuthenticatedCaller if anonymous => Decision::Deny,
|
||||
RouteGate::AuthenticatedCaller => Decision::Allow,
|
||||
}),
|
||||
RouteStage::LegacyNoCheck { .. } => Question::Decided(Decision::Allow),
|
||||
}
|
||||
}
|
||||
|
||||
async fn answer(&self, question: Question<'_>) -> Decision {
|
||||
match question {
|
||||
Question::Decided(decision) => decision,
|
||||
Question::AskCore(row, facts) => core_decision(self.core.decide(row, &facts).await),
|
||||
}
|
||||
}
|
||||
|
||||
fn ask<'a>(&self, context: &GatewayRequestContext<'a>, request: &AuthzRequest<'a>) -> Question<'a> {
|
||||
self.question(request.operation, request.identity.is_none(), || route_facts(context, request))
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads the core's answer as a decision. Only an explicit denial is a `Deny`; a failure to
|
||||
/// answer is `Indeterminate` and never an allowance.
|
||||
fn core_decision(answer: S3Result<()>) -> Decision {
|
||||
match answer {
|
||||
Ok(()) => Decision::Allow,
|
||||
Err(error) if error.code() == &S3ErrorCode::AccessDenied => Decision::Deny,
|
||||
Err(_) => Decision::Indeterminate,
|
||||
}
|
||||
}
|
||||
|
||||
fn route_facts<'a>(context: &GatewayRequestContext<'a>, request: &AuthzRequest<'a>) -> RouteFacts<'a> {
|
||||
let host = context.server_extensions();
|
||||
RouteFacts {
|
||||
access_key: request.identity.map(|identity| identity.access_key_id()),
|
||||
bucket: request.bucket.map(|bucket| bucket.as_str()),
|
||||
key: request.key.map(|key| key.as_str()),
|
||||
version_id: request.version_id,
|
||||
headers: context.headers().map(|accepted| {
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in accepted.iter_raw() {
|
||||
headers.append(name.clone(), value.clone());
|
||||
}
|
||||
headers
|
||||
}),
|
||||
request_line: host.get::<RequestLine>(),
|
||||
client_info: host.get::<ClientInfo>(),
|
||||
remote_addr: host
|
||||
.get::<Option<RemoteAddr>>()
|
||||
.and_then(|peer| peer.as_ref().map(|peer| peer.0)),
|
||||
request_context: host.get::<RequestContext>(),
|
||||
}
|
||||
}
|
||||
|
||||
impl Authorizer for RustfsAuthorizer {
|
||||
fn authorize_route<'a>(
|
||||
&'a self,
|
||||
context: &'a GatewayRequestContext<'a>,
|
||||
request: &'a AuthzRequest<'a>,
|
||||
) -> BoxFuture<'a, Decision> {
|
||||
let question = self.ask(context, request);
|
||||
Box::pin(self.answer(question))
|
||||
}
|
||||
|
||||
fn authorize_input<'a>(
|
||||
&'a self,
|
||||
context: &'a GatewayRequestContext<'a>,
|
||||
request: &'a InputAuthzRequest<'a>,
|
||||
) -> BoxFuture<'a, InputDecisions> {
|
||||
// `decide_all` visits every derived resource and then the visibility question, in that
|
||||
// order; each is read from the table now and answered below, where the core is awaited.
|
||||
let stage = self.ask(context, request.route());
|
||||
let mut derived: Vec<Question<'a>> = request
|
||||
.resources()
|
||||
.iter()
|
||||
.map(|resource| self.ask(context, resource))
|
||||
.collect();
|
||||
if let Some(visibility) = request.visibility() {
|
||||
derived.push(self.ask(context, visibility));
|
||||
}
|
||||
Box::pin(async move {
|
||||
let stage = self.answer(stage).await;
|
||||
let mut answers = Vec::with_capacity(derived.len());
|
||||
for question in derived {
|
||||
answers.push(self.answer(question).await);
|
||||
}
|
||||
let mut answers = answers.into_iter();
|
||||
request.decide_all(stage, |_| answers.next().unwrap_or(Decision::Deny))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Mutex;
|
||||
|
||||
/// The rows documented as `DeferToHandlerHook` in this file's header.
|
||||
const DOCUMENTED_DEFERRED_ROWS: &[&str] = &["ListBuckets"];
|
||||
/// The rows documented as `LegacyNoCheck` in this file's header.
|
||||
const DOCUMENTED_NO_CHECK_ROWS: &[&str] = &[];
|
||||
|
||||
static PLAIN_TABLE: &[AuthorizationRow] = &[
|
||||
AuthorizationRow {
|
||||
operation: "HeadBucket",
|
||||
action: Action::S3Action(S3Action::ListBucketAction),
|
||||
resource: ResourceForm::Bucket,
|
||||
route_stage: RouteStage::RouteDecidable,
|
||||
body_inputs: &[],
|
||||
extra_checks: ExtraChecks::None,
|
||||
},
|
||||
AuthorizationRow {
|
||||
operation: "GetObjectTorrent",
|
||||
action: Action::S3Action(S3Action::GetObjectAction),
|
||||
resource: ResourceForm::Object,
|
||||
route_stage: RouteStage::LegacyNoCheck {
|
||||
reason: "the legacy hook is empty",
|
||||
},
|
||||
body_inputs: &[],
|
||||
extra_checks: ExtraChecks::None,
|
||||
},
|
||||
];
|
||||
|
||||
/// A core that gives one scripted answer and records what it was asked.
|
||||
struct ScriptedCore {
|
||||
answer: fn() -> S3Result<()>,
|
||||
asked: Mutex<Vec<(&'static str, Action, Option<String>)>>,
|
||||
}
|
||||
|
||||
impl ScriptedCore {
|
||||
fn answering(answer: fn() -> S3Result<()>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
answer,
|
||||
asked: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl DecisionCore for ScriptedCore {
|
||||
fn decide<'a>(&'a self, row: &'a AuthorizationRow, facts: &'a RouteFacts<'a>) -> BoxFuture<'a, S3Result<()>> {
|
||||
self.asked
|
||||
.lock()
|
||||
.expect("core lock")
|
||||
.push((row.operation, row.action, facts.access_key.map(str::to_owned)));
|
||||
let answer = (self.answer)();
|
||||
Box::pin(async move { answer })
|
||||
}
|
||||
}
|
||||
|
||||
fn denied() -> S3Result<()> {
|
||||
Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"))
|
||||
}
|
||||
|
||||
fn unavailable() -> S3Result<()> {
|
||||
Err(S3Error::with_message(S3ErrorCode::InternalError, "the policy store is unavailable"))
|
||||
}
|
||||
|
||||
/// One request as a test asks about it.
|
||||
struct Asked<'a> {
|
||||
operation: &'a str,
|
||||
facts: RouteFacts<'a>,
|
||||
}
|
||||
|
||||
fn facts<'a>(operation: &'a str, access_key: Option<&'a str>) -> Asked<'a> {
|
||||
Asked {
|
||||
operation,
|
||||
facts: RouteFacts {
|
||||
access_key,
|
||||
bucket: None,
|
||||
key: None,
|
||||
version_id: None,
|
||||
headers: None,
|
||||
request_line: None,
|
||||
client_info: None,
|
||||
remote_addr: None,
|
||||
request_context: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn decision(table: &'static [AuthorizationRow], core: Arc<dyn DecisionCore>, asked: Asked<'_>) -> Decision {
|
||||
let authorizer = RustfsAuthorizer::new(table, core);
|
||||
let anonymous = asked.facts.access_key.is_none();
|
||||
let question = authorizer.question(asked.operation, anonymous, || asked.facts);
|
||||
authorizer.answer(question).await
|
||||
}
|
||||
|
||||
/// The real decision core of a server that has no object store yet.
|
||||
fn owned_core() -> Arc<dyn DecisionCore> {
|
||||
Arc::new(OwnedDecisionCore::new(ServerContextSlot::new()))
|
||||
}
|
||||
|
||||
fn head_bucket() -> RequestLine {
|
||||
RequestLine {
|
||||
method: Method::HEAD,
|
||||
uri: Uri::from_static("/bucket"),
|
||||
}
|
||||
}
|
||||
|
||||
fn rows_where(predicate: fn(&RouteStage) -> bool) -> BTreeSet<&'static str> {
|
||||
AUTHORIZATION_TABLE
|
||||
.iter()
|
||||
.filter(|row| predicate(&row.route_stage))
|
||||
.map(|row| row.operation)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_operation_without_a_row_is_denied() {
|
||||
let core = ScriptedCore::answering(|| Ok(()));
|
||||
|
||||
for operation in ["GetObject", "PutObject", "", "listbuckets"] {
|
||||
let decided = decision(AUTHORIZATION_TABLE, core.clone(), facts(operation, Some("AKIDUSER0001"))).await;
|
||||
assert_eq!(decided, Decision::Deny, "{operation:?}");
|
||||
}
|
||||
assert!(core.asked.lock().expect("core lock").is_empty(), "an unmapped operation asks nothing");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_core_that_cannot_answer_is_indeterminate_never_an_allowance() {
|
||||
let decided = decision(
|
||||
PLAIN_TABLE,
|
||||
ScriptedCore::answering(unavailable),
|
||||
facts("HeadBucket", Some("AKIDUSER0001")),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(decided, Decision::Indeterminate);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_owned_core_without_a_store_cannot_answer_and_never_allows() {
|
||||
let line = head_bucket();
|
||||
let mut asked = facts("HeadBucket", None);
|
||||
asked.facts.bucket = Some("bucket");
|
||||
asked.facts.headers = Some(HeaderMap::new());
|
||||
asked.facts.request_line = Some(&line);
|
||||
|
||||
assert_eq!(decision(PLAIN_TABLE, owned_core(), asked).await, Decision::Indeterminate);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_owned_core_refuses_a_request_the_host_did_not_describe() {
|
||||
let line = head_bucket();
|
||||
|
||||
let mut without_a_line = facts("HeadBucket", None);
|
||||
without_a_line.facts.bucket = Some("bucket");
|
||||
without_a_line.facts.headers = Some(HeaderMap::new());
|
||||
assert_eq!(decision(PLAIN_TABLE, owned_core(), without_a_line).await, Decision::Indeterminate);
|
||||
|
||||
let mut without_headers = facts("HeadBucket", None);
|
||||
without_headers.facts.bucket = Some("bucket");
|
||||
without_headers.facts.request_line = Some(&line);
|
||||
assert_eq!(decision(PLAIN_TABLE, owned_core(), without_headers).await, Decision::Indeterminate);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_owned_core_refuses_a_bucket_row_whose_request_names_no_bucket() {
|
||||
let line = head_bucket();
|
||||
let mut asked = facts("HeadBucket", None);
|
||||
asked.facts.headers = Some(HeaderMap::new());
|
||||
asked.facts.request_line = Some(&line);
|
||||
|
||||
assert_eq!(decision(PLAIN_TABLE, owned_core(), asked).await, Decision::Indeterminate);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_core_denial_is_a_denial() {
|
||||
let decided = decision(PLAIN_TABLE, ScriptedCore::answering(denied), facts("HeadBucket", Some("AKIDUSER0001"))).await;
|
||||
|
||||
assert_eq!(decided, Decision::Deny);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_route_decidable_row_asks_the_core_its_legacy_action_for_the_verified_caller() {
|
||||
let core = ScriptedCore::answering(|| Ok(()));
|
||||
|
||||
let decided = decision(PLAIN_TABLE, core.clone(), facts("HeadBucket", Some("AKIDUSER0001"))).await;
|
||||
|
||||
assert_eq!(decided, Decision::Allow);
|
||||
assert_eq!(
|
||||
*core.asked.lock().expect("core lock"),
|
||||
[(
|
||||
"HeadBucket",
|
||||
Action::S3Action(S3Action::ListBucketAction),
|
||||
Some("AKIDUSER0001".to_owned())
|
||||
)]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_deferred_row_refuses_an_anonymous_caller_without_asking_the_core() {
|
||||
let core = ScriptedCore::answering(|| Ok(()));
|
||||
|
||||
assert_eq!(
|
||||
decision(AUTHORIZATION_TABLE, core.clone(), facts("ListBuckets", None)).await,
|
||||
Decision::Deny
|
||||
);
|
||||
assert_eq!(
|
||||
decision(AUTHORIZATION_TABLE, core.clone(), facts("ListBuckets", Some("AKIDUSER0001"))).await,
|
||||
Decision::Allow,
|
||||
"a verified caller passes to the hook"
|
||||
);
|
||||
assert!(
|
||||
core.asked.lock().expect("core lock").is_empty(),
|
||||
"a deferred row decides nothing through the core"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_no_check_row_is_allowed_only_because_its_row_says_so() {
|
||||
let core = ScriptedCore::answering(denied);
|
||||
|
||||
assert_eq!(
|
||||
decision(PLAIN_TABLE, core.clone(), facts("GetObjectTorrent", None)).await,
|
||||
Decision::Allow
|
||||
);
|
||||
assert_eq!(decision(AUTHORIZATION_TABLE, core, facts("GetObjectTorrent", None)).await, Decision::Deny);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_deferred_rows_are_exactly_the_documented_ones() {
|
||||
let deferred = rows_where(|stage| matches!(stage, RouteStage::DeferToHandlerHook { .. }));
|
||||
|
||||
assert_eq!(deferred, DOCUMENTED_DEFERRED_ROWS.iter().copied().collect::<BTreeSet<_>>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_no_check_rows_are_exactly_the_documented_ones() {
|
||||
let unchecked = rows_where(|stage| matches!(stage, RouteStage::LegacyNoCheck { .. }));
|
||||
|
||||
assert_eq!(unchecked, DOCUMENTED_NO_CHECK_ROWS.iter().copied().collect::<BTreeSet<_>>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_row_states_every_column_and_names_one_operation() {
|
||||
let mut seen = BTreeSet::new();
|
||||
for row in AUTHORIZATION_TABLE {
|
||||
assert!(seen.insert(row.operation), "{} has two rows", row.operation);
|
||||
assert!(!row.operation.is_empty());
|
||||
match row.route_stage {
|
||||
RouteStage::DeferToHandlerHook { reason, .. } | RouteStage::LegacyNoCheck { reason } => {
|
||||
assert!(!reason.is_empty(), "{} defers without a reason", row.operation);
|
||||
}
|
||||
RouteStage::RouteDecidable => {}
|
||||
}
|
||||
match row.extra_checks {
|
||||
ExtraChecks::Performed(checks) => {
|
||||
assert!(!checks.is_empty(), "{} lists no extra check; say `ExtraChecks::None`", row.operation);
|
||||
assert!(checks.iter().all(|check| !check.is_empty()), "{}", row.operation);
|
||||
}
|
||||
ExtraChecks::None => {}
|
||||
}
|
||||
assert!(row.body_inputs.iter().all(|input| !input.is_empty()), "{}", row.operation);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_buckets_asks_the_legacy_action_on_the_service() {
|
||||
let row = AUTHORIZATION_TABLE
|
||||
.iter()
|
||||
.find(|row| row.operation == "ListBuckets")
|
||||
.expect("ListBuckets is registered");
|
||||
|
||||
assert_eq!(row.action, Action::S3Action(S3Action::ListAllMyBucketsAction));
|
||||
assert_eq!(row.resource, ResourceForm::Service);
|
||||
assert_eq!(
|
||||
row.route_stage,
|
||||
RouteStage::DeferToHandlerHook {
|
||||
gate: RouteGate::AuthenticatedCaller,
|
||||
reason: "the use case decides per caller from the bucket inventory",
|
||||
}
|
||||
);
|
||||
assert!(row.body_inputs.is_empty());
|
||||
assert!(matches!(row.extra_checks, ExtraChecks::Performed(_)));
|
||||
}
|
||||
}
|
||||
@@ -1,561 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! The assembled gateway stack, driven through [`GatewayStack::call`] exactly as `server/http.rs`
|
||||
//! drives it. Requests that need object storage are in `rustfs/tests/embedded_gateway_stack_test.rs`.
|
||||
|
||||
use super::authorizer::AUTHORIZATION_TABLE;
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, Method, StatusCode};
|
||||
use http_body_util::{BodyExt, Empty};
|
||||
use rustfs_gateway::{AuthzRequest, ClientFacts, Decision, InputAuthzRequest, InputDecisions, NO_DEADLINE};
|
||||
use std::collections::BTreeSet;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Mutex;
|
||||
use std::time::Instant;
|
||||
|
||||
const ROOT_ACCESS_KEY: &str = "rustfsgatewayroot";
|
||||
const ROOT_SECRET_KEY: &str = "rustfs-gateway-root-secret";
|
||||
const REQUEST_ID: &str = "7c9e6679-7425-40de-944b-e07fc1f90ae7";
|
||||
const PEER: ([u8; 4], u16) = ([10, 0, 0, 7], 41000);
|
||||
const CLIENT: [u8; 4] = [203, 0, 113, 9];
|
||||
const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
||||
|
||||
fn assembly() -> GatewayAssembly {
|
||||
GatewayAssembly {
|
||||
server_ctx: ServerContextSlot::new(),
|
||||
root_access_key: ROOT_ACCESS_KEY.to_owned(),
|
||||
root_secret_key: ROOT_SECRET_KEY.to_owned(),
|
||||
region: Some("us-east-1".to_owned()),
|
||||
server_domains: Vec::new(),
|
||||
body_read_timeout: Duration::from_secs(300),
|
||||
keystone_enabled: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// An IAM whose readiness and answer are scripted.
|
||||
struct ScriptedIam {
|
||||
ready: bool,
|
||||
}
|
||||
|
||||
impl IamKeyLookup for ScriptedIam {
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
|
||||
Box::pin(async { Ok((None, false)) })
|
||||
}
|
||||
}
|
||||
|
||||
impl IamPrincipalLookup for ScriptedIam {
|
||||
fn group_memberships(&self, _user: &str) -> Vec<String> {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// A settable live root, standing in for the server's runtime credentials.
|
||||
#[derive(Clone)]
|
||||
struct LiveRoot(Arc<Mutex<Option<Arc<RootCredentials>>>>);
|
||||
|
||||
impl LiveRoot {
|
||||
fn set(&self, root: RootCredentials) {
|
||||
*self.0.lock().expect("root lock") = Some(Arc::new(root));
|
||||
}
|
||||
|
||||
fn clear(&self) {
|
||||
*self.0.lock().expect("root lock") = None;
|
||||
}
|
||||
|
||||
fn source(&self) -> Arc<dyn RootSource> {
|
||||
let cell = Arc::clone(&self.0);
|
||||
Arc::new(move || cell.lock().expect("root lock").clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// A live root holding the test root.
|
||||
fn root() -> LiveRoot {
|
||||
let root = RootCredentials::new(ROOT_ACCESS_KEY, ROOT_SECRET_KEY, ROOT_SECRET_KEY).expect("a usable root");
|
||||
LiveRoot(Arc::new(Mutex::new(Some(Arc::new(root)))))
|
||||
}
|
||||
|
||||
/// The authorizer production installs: the registered table over the shared decision core.
|
||||
fn production_authorizer() -> RustfsAuthorizer {
|
||||
RustfsAuthorizer::new(AUTHORIZATION_TABLE, Arc::new(OwnedDecisionCore::new(ServerContextSlot::new())))
|
||||
}
|
||||
|
||||
/// The stack as production assembles it, over an IAM that knows no key but the root's.
|
||||
fn stack_over(iam: ScriptedIam) -> GatewayStack {
|
||||
let authorizer = production_authorizer();
|
||||
GatewayStack::assemble_with(assembly(), Arc::new(iam), root().source(), authorizer).expect("the stack assembles")
|
||||
}
|
||||
|
||||
fn request(method: Method, target: &str) -> http::request::Builder {
|
||||
Request::builder().method(method).uri(target).header("host", "localhost")
|
||||
}
|
||||
|
||||
/// A well-formed SigV4 header signature for `access_key` whose signature is not the right one.
|
||||
fn signed_with_a_wrong_signature(builder: http::request::Builder, access_key: &str) -> http::request::Builder {
|
||||
let now = jiff::Timestamp::now().strftime("%Y%m%dT%H%M%SZ").to_string();
|
||||
let date = &now[..8];
|
||||
builder
|
||||
.header("x-amz-date", now.as_str())
|
||||
.header("x-amz-content-sha256", EMPTY_SHA256)
|
||||
.header(
|
||||
"authorization",
|
||||
format!(
|
||||
"AWS4-HMAC-SHA256 Credential={access_key}/{date}/us-east-1/s3/aws4_request, \
|
||||
SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature={}",
|
||||
"0".repeat(64)
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
async fn answer(stack: &GatewayStack, builder: http::request::Builder) -> (StatusCode, HeaderMap, String) {
|
||||
let response = stack.call(builder.body(Empty::<Bytes>::new()).expect("static request")).await;
|
||||
let (parts, body) = response.into_parts();
|
||||
let body = body.collect().await.expect("the answer is read").to_bytes();
|
||||
(parts.status, parts.headers, String::from_utf8_lossy(&body).into_owned())
|
||||
}
|
||||
|
||||
fn rustfs_request_context(request_id: &str) -> RequestContext {
|
||||
RequestContext {
|
||||
request_id: request_id.to_owned(),
|
||||
x_amz_request_id: request_id.to_owned(),
|
||||
trace_id: None,
|
||||
span_id: None,
|
||||
start_time: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_registered_operation_has_an_authorization_row_and_every_row_an_operation() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let registered: BTreeSet<&str> = stack.registered_operations().into_iter().collect();
|
||||
let mapped: BTreeSet<&str> = AUTHORIZATION_TABLE.iter().map(|row| row.operation).collect();
|
||||
|
||||
assert_eq!(registered, mapped);
|
||||
assert_eq!(registered, BTreeSet::from(["ListBuckets"]), "the walking skeleton serves one operation");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn production_assembly_registers_what_the_test_assembly_does() {
|
||||
let stack = GatewayStack::assemble(assembly()).expect("the stack assembles");
|
||||
|
||||
assert_eq!(stack.registered_operations(), ["ListBuckets"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unusable_region_fails_the_assembly() {
|
||||
let refused = GatewayStack::assemble(GatewayAssembly {
|
||||
region: Some("Not A Region".to_owned()),
|
||||
..assembly()
|
||||
});
|
||||
|
||||
assert!(matches!(refused, Err(GatewayAssemblyError::Region { .. })), "{:?}", refused.err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unusable_root_access_key_fails_the_assembly_without_naming_the_secret() {
|
||||
let refused = GatewayStack::assemble(GatewayAssembly {
|
||||
root_access_key: String::new(),
|
||||
..assembly()
|
||||
});
|
||||
|
||||
let error = refused.err().expect("an empty access key verifies nothing");
|
||||
assert!(
|
||||
matches!(error, GatewayAssemblyError::RootCredentials(RootCredentialsError::InvalidAccessKey)),
|
||||
"{error:?}"
|
||||
);
|
||||
for rendered in [error.to_string(), format!("{error:?}")] {
|
||||
assert!(!rendered.contains(ROOT_SECRET_KEY), "{rendered}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_or_short_root_secret_fails_the_assembly() {
|
||||
for weak in ["", "1234567"] {
|
||||
let refused = GatewayStack::assemble(GatewayAssembly {
|
||||
root_secret_key: weak.to_owned(),
|
||||
..assembly()
|
||||
});
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
refused.as_ref().err(),
|
||||
Some(GatewayAssemblyError::RootCredentials(RootCredentialsError::WeakSecretKey))
|
||||
),
|
||||
"{} bytes",
|
||||
weak.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_root_change_reaches_the_next_request_and_a_withdrawn_root_signs_nothing() {
|
||||
const ROTATED_ACCESS_KEY: &str = "rotatedgatewayroot";
|
||||
let live = root();
|
||||
let authorizer = production_authorizer();
|
||||
let stack = GatewayStack::assemble_with(assembly(), Arc::new(ScriptedIam { ready: true }), live.source(), authorizer)
|
||||
.expect("the stack assembles");
|
||||
let signed_as = |access_key: &str| signed_with_a_wrong_signature(request(Method::GET, "/"), access_key);
|
||||
|
||||
let (_, _, known) = answer(&stack, signed_as(ROOT_ACCESS_KEY)).await;
|
||||
assert!(known.contains("<Code>SignatureDoesNotMatch</Code>"), "the root key is known: {known}");
|
||||
|
||||
// The root access key changes: the old one is now just an unknown key, and the new one is
|
||||
// known to the very next request.
|
||||
live.set(RootCredentials::new(ROTATED_ACCESS_KEY, ROOT_SECRET_KEY, ROOT_SECRET_KEY).expect("a usable root"));
|
||||
let (status, _, retired) = answer(&stack, signed_as(ROOT_ACCESS_KEY)).await;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "{retired}");
|
||||
assert!(retired.contains("<Code>InvalidAccessKeyId</Code>"), "{retired}");
|
||||
let (_, _, rotated) = answer(&stack, signed_as(ROTATED_ACCESS_KEY)).await;
|
||||
assert!(
|
||||
rotated.contains("<Code>SignatureDoesNotMatch</Code>"),
|
||||
"the new root key is known: {rotated}"
|
||||
);
|
||||
|
||||
// Injected: a root source that returns nothing, so the provider returns an error for the
|
||||
// lookup. The key that was root a moment ago verifies nothing; the pinned gateway answers the
|
||||
// provider fault as an unknown key. Ledger entry R-068 (rustfs/backlog#2684): this status
|
||||
// becomes `500` when the pin includes rustfs/gateway#1585 and the assembly sets its switch.
|
||||
live.clear();
|
||||
let (status, _, withdrawn) = answer(&stack, signed_as(ROTATED_ACCESS_KEY)).await;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "{withdrawn}");
|
||||
assert!(withdrawn.contains("<Code>InvalidAccessKeyId</Code>"), "{withdrawn}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keystone_enabled_fails_the_assembly_and_names_both_settings() {
|
||||
let refused = GatewayStack::assemble(GatewayAssembly {
|
||||
keystone_enabled: true,
|
||||
..assembly()
|
||||
});
|
||||
|
||||
let error = refused.err().expect("the gateway stack is not served with Keystone enabled");
|
||||
assert!(matches!(error, GatewayAssemblyError::KeystoneEnabled), "{error:?}");
|
||||
let message = error.to_string();
|
||||
for named in ["RUSTFS_S3_STACK=gateway", "RUSTFS_KEYSTONE_ENABLE", "rustfs/gateway#1586"] {
|
||||
assert!(message.contains(named), "{named} is missing from: {message}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keystone_disabled_assembles() {
|
||||
GatewayStack::assemble(GatewayAssembly {
|
||||
keystone_enabled: false,
|
||||
..assembly()
|
||||
})
|
||||
.expect("the gateway stack assembles without Keystone");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keystone_is_read_from_configuration_and_an_unreadable_one_counts_as_enabled() {
|
||||
const ENABLE: &str = "RUSTFS_KEYSTONE_ENABLE";
|
||||
const AUTH_URL: &str = "RUSTFS_KEYSTONE_AUTH_URL";
|
||||
|
||||
temp_env::with_vars([(ENABLE, None::<&str>), (AUTH_URL, None)], || {
|
||||
assert!(!keystone_enabled_in_configuration(), "unset is off");
|
||||
});
|
||||
temp_env::with_vars([(ENABLE, Some("false")), (AUTH_URL, None)], || {
|
||||
assert!(!keystone_enabled_in_configuration(), "false is off");
|
||||
});
|
||||
temp_env::with_vars([(ENABLE, Some("true")), (AUTH_URL, Some("http://keystone.example.test:5000"))], || {
|
||||
assert!(keystone_enabled_in_configuration(), "true is on");
|
||||
});
|
||||
temp_env::with_vars([(ENABLE, Some("true")), (AUTH_URL, None)], || {
|
||||
assert!(keystone_enabled_in_configuration(), "on but unreadable is still on");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_handler_runs_under_a_total_deadline_and_the_body_keeps_the_rustfs_idle_bound() {
|
||||
let config = service_config(Duration::from_secs(300));
|
||||
|
||||
assert_eq!(config.handler_deadline(HandlerDeadlineClass::Standard), NO_DEADLINE);
|
||||
assert_eq!(config.handler_deadline(HandlerDeadlineClass::Extended), NO_DEADLINE);
|
||||
assert_eq!(config.commit_progress_deadline(), NO_DEADLINE);
|
||||
assert_eq!(config.request_body_deadlines().read_idle(), Duration::from_secs(300));
|
||||
assert_eq!(config.request_body_deadlines().first_byte(), Duration::from_secs(300));
|
||||
assert_eq!(config.request_body_deadlines().throughput_window(), NO_DEADLINE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_disabled_body_read_timeout_lifts_the_idle_bound_instead_of_arming_zero() {
|
||||
let deadlines = body_deadlines(Duration::ZERO);
|
||||
|
||||
assert_eq!(deadlines.read_idle(), NO_DEADLINE);
|
||||
assert_eq!(deadlines.first_byte(), NO_DEADLINE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_facts_come_from_what_rustfs_layers_attached() {
|
||||
let peer = SocketAddr::from(PEER);
|
||||
let mut extensions = Extensions::new();
|
||||
extensions.insert(Some(RemoteAddr(peer)));
|
||||
extensions.insert(ClientInfo::direct(SocketAddr::new(IpAddr::from(CLIENT), 443)));
|
||||
extensions.insert(NativeTlsTransport(true));
|
||||
extensions.insert(rustfs_request_context(REQUEST_ID));
|
||||
|
||||
attach_host_facts(&mut extensions);
|
||||
|
||||
assert_eq!(extensions.get::<TransportSecurity>(), Some(&TransportSecurity::Encrypted));
|
||||
assert_eq!(
|
||||
extensions.get::<ClientAddr>().map(|address| address.ip()),
|
||||
Some(IpAddr::from(CLIENT)),
|
||||
"the trusted-proxy-corrected address wins over the socket peer"
|
||||
);
|
||||
assert_eq!(
|
||||
extensions
|
||||
.get::<HostRequestId>()
|
||||
.map(|id| id.request_id().as_str().to_owned()),
|
||||
Some(REQUEST_ID.to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_facts_never_lean_towards_secure_or_invent_an_address() {
|
||||
let mut bare = Extensions::new();
|
||||
attach_host_facts(&mut bare);
|
||||
assert_eq!(bare.get::<TransportSecurity>(), Some(&TransportSecurity::Plaintext));
|
||||
assert!(bare.get::<ClientAddr>().is_none(), "no address is known");
|
||||
assert!(bare.get::<HostRequestId>().is_none(), "no identifier is known");
|
||||
|
||||
let mut plaintext = Extensions::new();
|
||||
plaintext.insert(Some(RemoteAddr(SocketAddr::from(PEER))));
|
||||
plaintext.insert(NativeTlsTransport(false));
|
||||
plaintext.insert(rustfs_request_context("not a request id"));
|
||||
attach_host_facts(&mut plaintext);
|
||||
assert_eq!(plaintext.get::<TransportSecurity>(), Some(&TransportSecurity::Plaintext));
|
||||
assert_eq!(
|
||||
plaintext.get::<ClientAddr>().map(|address| address.ip()),
|
||||
Some(IpAddr::from(PEER.0)),
|
||||
"without a proxy the socket peer is the client"
|
||||
);
|
||||
assert!(
|
||||
plaintext.get::<HostRequestId>().is_none(),
|
||||
"an identifier outside the alphabet is not handed over"
|
||||
);
|
||||
|
||||
let mut peerless = Extensions::new();
|
||||
peerless.insert(None::<RemoteAddr>);
|
||||
attach_host_facts(&mut peerless);
|
||||
assert!(peerless.get::<ClientAddr>().is_none());
|
||||
}
|
||||
|
||||
/// Records what the gateway hands its authorizer about the client and the request line, and
|
||||
/// refuses.
|
||||
struct RecordingAuthorizer {
|
||||
seen: Arc<Mutex<Option<Option<ClientFacts>>>>,
|
||||
line: Arc<Mutex<Option<RequestLine>>>,
|
||||
}
|
||||
|
||||
impl Authorizer for RecordingAuthorizer {
|
||||
fn authorize_route<'a>(
|
||||
&'a self,
|
||||
context: &'a rustfs_gateway::RequestContext<'a>,
|
||||
_request: &'a AuthzRequest<'a>,
|
||||
) -> BoxFuture<'a, Decision> {
|
||||
*self.seen.lock().expect("recording lock") = Some(context.client().copied());
|
||||
*self.line.lock().expect("recording lock") = context.server_extensions().get::<RequestLine>().cloned();
|
||||
Box::pin(async { Decision::Deny })
|
||||
}
|
||||
|
||||
fn authorize_input<'a>(
|
||||
&'a self,
|
||||
_context: &'a rustfs_gateway::RequestContext<'a>,
|
||||
request: &'a InputAuthzRequest<'a>,
|
||||
) -> BoxFuture<'a, InputDecisions> {
|
||||
let decisions = request.decide_all(Decision::Deny, |_| Decision::Deny);
|
||||
Box::pin(async move { decisions })
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_authorizer_sees_the_client_facts_the_host_attached() {
|
||||
let seen = Arc::new(Mutex::new(None));
|
||||
let line = Arc::new(Mutex::new(None));
|
||||
let authorizer = RecordingAuthorizer {
|
||||
seen: Arc::clone(&seen),
|
||||
line: Arc::clone(&line),
|
||||
};
|
||||
let stack = GatewayStack::assemble_with(assembly(), Arc::new(ScriptedIam { ready: true }), root().source(), authorizer)
|
||||
.expect("assembles");
|
||||
let proxied = request(Method::GET, "/?prefix=probe")
|
||||
.extension(Some(RemoteAddr(SocketAddr::from(PEER))))
|
||||
.extension(ClientInfo::direct(SocketAddr::new(IpAddr::from(CLIENT), 443)))
|
||||
.extension(NativeTlsTransport(true));
|
||||
|
||||
let (status, _, _) = answer(&stack, proxied).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN);
|
||||
let facts = seen
|
||||
.lock()
|
||||
.expect("recording lock")
|
||||
.take()
|
||||
.expect("the authorizer was asked")
|
||||
.expect("the host attached client facts");
|
||||
assert!(facts.transport_secure, "the listener terminated TLS");
|
||||
assert_eq!(facts.client_ip, Some(IpAddr::from(CLIENT)));
|
||||
assert_eq!(facts.source_ip(), Some(IpAddr::from(CLIENT)));
|
||||
let line = line
|
||||
.lock()
|
||||
.expect("recording lock")
|
||||
.take()
|
||||
.expect("the host attached the request line");
|
||||
assert_eq!(line.method, Method::GET);
|
||||
assert_eq!(line.uri.path(), "/");
|
||||
assert_eq!(line.uri.query(), Some("prefix=probe"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_authorizer_sees_a_cleartext_transport_as_cleartext() {
|
||||
let seen = Arc::new(Mutex::new(None));
|
||||
let line = Arc::new(Mutex::new(None));
|
||||
let authorizer = RecordingAuthorizer {
|
||||
seen: Arc::clone(&seen),
|
||||
line: Arc::clone(&line),
|
||||
};
|
||||
let stack = GatewayStack::assemble_with(assembly(), Arc::new(ScriptedIam { ready: true }), root().source(), authorizer)
|
||||
.expect("assembles");
|
||||
|
||||
let (status, _, _) = answer(&stack, request(Method::GET, "/").extension(NativeTlsTransport(false))).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN);
|
||||
let facts = seen
|
||||
.lock()
|
||||
.expect("recording lock")
|
||||
.take()
|
||||
.expect("the authorizer was asked")
|
||||
.expect("the transport declaration alone makes the facts present");
|
||||
assert!(!facts.transport_secure);
|
||||
assert_eq!(facts.client_ip, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_anonymous_list_buckets_is_denied() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (status, _, body) = answer(&stack, request(Method::GET, "/")).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN);
|
||||
assert!(body.contains("<Code>AccessDenied</Code>"), "{body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unregistered_operation_is_not_implemented() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
for (method, target) in [
|
||||
(Method::GET, "/bucket/key"),
|
||||
(Method::PUT, "/bucket"),
|
||||
(Method::DELETE, "/bucket/key"),
|
||||
] {
|
||||
let (status, _, body) = answer(&stack, request(method.clone(), target)).await;
|
||||
|
||||
assert_eq!(status, StatusCode::NOT_IMPLEMENTED, "{method} {target}: {body}");
|
||||
assert!(body.contains("<Code>NotImplemented</Code>"), "{method} {target}: {body}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_wrong_signature_is_a_signature_mismatch() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), ROOT_ACCESS_KEY)).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN);
|
||||
assert!(body.contains("<Code>SignatureDoesNotMatch</Code>"), "{body}");
|
||||
assert!(!body.contains(ROOT_SECRET_KEY), "{body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unknown_access_key_is_refused_as_the_caller_s_fault() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), "AKIDUNKNOWN0001")).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "{body}");
|
||||
assert!(body.contains("<Code>InvalidAccessKeyId</Code>"), "{body}");
|
||||
}
|
||||
|
||||
/// Injected: a credential provider that returns an error for the lookup of a non-root key.
|
||||
///
|
||||
/// The pinned gateway's SigV4 authenticator answers a provider fault exactly as it answers an
|
||||
/// unknown key, `403 InvalidAccessKeyId`; the legacy stack answers a failed lookup `500`. Ledger
|
||||
/// entry R-068 (rustfs/backlog#2684): rustfs/gateway#1585 adds an opt-in switch that answers the
|
||||
/// fault as one. When the pin includes it and the assembly sets it, this test expects `500`.
|
||||
/// Until then the status is pinned, and what holds on both stacks is that nothing is served.
|
||||
#[tokio::test]
|
||||
async fn a_credential_lookup_that_fails_is_refused_and_never_served() {
|
||||
let stack = stack_over(ScriptedIam { ready: false });
|
||||
|
||||
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), "AKIDUSER0001")).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "{body}");
|
||||
assert!(body.contains("<Code>InvalidAccessKeyId</Code>"), "{body}");
|
||||
assert!(!body.contains("ListAllMyBucketsResult"), "{body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_root_key_verifies_while_iam_is_unavailable() {
|
||||
let stack = stack_over(ScriptedIam { ready: false });
|
||||
|
||||
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), ROOT_ACCESS_KEY)).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "the root secret is known without IAM: {body}");
|
||||
assert!(body.contains("<Code>SignatureDoesNotMatch</Code>"), "{body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_answer_carries_the_request_identifier_rustfs_assigned() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (_, headers, _) = answer(&stack, request(Method::GET, "/").extension(rustfs_request_context(REQUEST_ID))).await;
|
||||
|
||||
assert_eq!(headers.get("x-amz-request-id").and_then(|value| value.to_str().ok()), Some(REQUEST_ID));
|
||||
assert_eq!(headers.get("x-request-id").and_then(|value| value.to_str().ok()), Some(REQUEST_ID));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn without_a_rustfs_identifier_the_gateway_mints_its_own() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (_, headers, _) = answer(&stack, request(Method::GET, "/")).await;
|
||||
|
||||
let minted = headers
|
||||
.get("x-amz-request-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("every answer names its request");
|
||||
assert!(!minted.is_empty());
|
||||
assert_ne!(minted, REQUEST_ID);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_gateway_writes_no_cors_header() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
let browser = |builder: http::request::Builder| builder.header("origin", "https://app.example.test");
|
||||
|
||||
for (method, target) in [(Method::GET, "/"), (Method::GET, "/bucket/key"), (Method::PUT, "/bucket/key")] {
|
||||
let (_, headers, _) = answer(&stack, browser(request(method.clone(), target))).await;
|
||||
|
||||
let cors: Vec<&str> = headers
|
||||
.keys()
|
||||
.map(http::HeaderName::as_str)
|
||||
.filter(|name| name.starts_with("access-control-"))
|
||||
.collect();
|
||||
assert!(cors.is_empty(), "{method} {target}: {cors:?}");
|
||||
}
|
||||
}
|
||||
@@ -18,9 +18,6 @@ use crate::app::object::request_body::{BodyReadControl, ObservedBody};
|
||||
use crate::auth::IAMAuth;
|
||||
use crate::auth_keystone;
|
||||
use crate::config;
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
use crate::server::gateway_stack::{GatewayAssembly, GatewayStack, keystone_enabled_in_configuration};
|
||||
use crate::server::stack::{S3StackService, select_stack};
|
||||
use crate::server::{
|
||||
Http12ShutdownOutcome, ReadinessGateLayer, RemoteAddr, ShutdownHandle,
|
||||
compress::{HttpCompressionConfig, PathAwareHttpCompressionPredicate, PathCategoryInjectionLayer},
|
||||
@@ -262,7 +259,6 @@ const EVENT_HTTP_TRANSPORT_FAILED: &str = "http_transport_failed";
|
||||
const EVENT_SOCKET_FALLBACK: &str = "socket_fallback";
|
||||
const EVENT_HTTP_BIND_FAILED: &str = "http_bind_failed";
|
||||
const EVENT_HTTP_STARTUP_ENDPOINTS: &str = "http_startup_endpoints";
|
||||
const EVENT_S3_STACK_SELECTED: &str = "s3_stack_selected";
|
||||
const EVENT_HTTP_HOST_ROUTING: &str = "http_host_routing";
|
||||
const EVENT_HTTP_COMPRESSION_STATE: &str = "http_compression_state";
|
||||
const EVENT_API_RATE_LIMIT_STATE: &str = "api_rate_limit_state";
|
||||
@@ -1464,41 +1460,6 @@ fn control_request_body(body: s3s::Body) -> ControlRequestBody {
|
||||
ControlRequestBody(body)
|
||||
}
|
||||
|
||||
/// The S3 service mounted under the middleware: the legacy stack or the gateway stack, selected
|
||||
/// at startup (`server/stack.rs`).
|
||||
type S3EdgeService = S3StackService<S3Service, GatewayEdgeService>;
|
||||
|
||||
/// A build without the `gateway-stack` feature has no gateway stack to select, so this side of
|
||||
/// [`S3EdgeService`] is never constructed.
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
type GatewayEdgeService = S3Service;
|
||||
|
||||
/// The gateway stack where the legacy service is mounted: the same response and error types, so
|
||||
/// the layers around it do not change with the stack.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
#[derive(Clone)]
|
||||
struct GatewayEdgeService(GatewayStack);
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
impl<B> Service<HttpRequest<B>> for GatewayEdgeService
|
||||
where
|
||||
B: http_body::Body<Data = Bytes> + Send + 'static,
|
||||
B::Error: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
type Response = Response<s3s::Body>;
|
||||
type Error = s3s::HttpError;
|
||||
type Future = Pin<Box<dyn Future<Output = std::result::Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, request: HttpRequest<B>) -> Self::Future {
|
||||
let answer = self.0.call(request);
|
||||
Box::pin(async move { Ok(answer.await.map(s3s::Body::http_body_unsync)) })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ControlBodyDigestService<S> {
|
||||
inner: S,
|
||||
@@ -2491,48 +2452,6 @@ async fn start_http_server_inner(
|
||||
rustfs_config::ENV_HTTP_REQUEST_BODY_READ_TIMEOUT,
|
||||
rustfs_config::DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT,
|
||||
));
|
||||
|
||||
// The S3 stack is selected once, for the whole server. A gateway stack that cannot be
|
||||
// assembled fails startup here; the legacy stack is never started in its place.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
let assemble_gateway = || -> Result<GatewayEdgeService> {
|
||||
let stack = GatewayStack::assemble(GatewayAssembly {
|
||||
server_ctx: Arc::clone(&server_ctx),
|
||||
root_access_key: config.access_key.clone(),
|
||||
root_secret_key: config.secret_key.clone(),
|
||||
region: config.region.clone(),
|
||||
server_domains: vh_domain_hosts.to_vec(),
|
||||
body_read_timeout: http_request_body_read_timeout,
|
||||
keystone_enabled: keystone_enabled_in_configuration(),
|
||||
})
|
||||
.map_err(Error::other)?;
|
||||
info!(
|
||||
event = EVENT_S3_STACK_SELECTED,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
stack = "gateway",
|
||||
registered_operations = stack.registered_operations().len(),
|
||||
"S3 stack selected"
|
||||
);
|
||||
Ok(GatewayEdgeService(stack))
|
||||
};
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
let assemble_gateway = || {
|
||||
Err(Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"the gateway S3 stack is not built into this binary",
|
||||
))
|
||||
};
|
||||
let s3_service: S3EdgeService = select_stack(config.s3_stack, s3_service, assemble_gateway)?;
|
||||
if s3_service.selected() == crate::config::S3Stack::Legacy {
|
||||
info!(
|
||||
event = EVENT_S3_STACK_SELECTED,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
stack = "legacy",
|
||||
"S3 stack selected"
|
||||
);
|
||||
}
|
||||
#[cfg(feature = "http3")]
|
||||
let http3_server = if let Some(server_config) = http3_server_config {
|
||||
let h3_s3_service = s3_service.clone();
|
||||
@@ -2970,7 +2889,7 @@ async fn start_http_server_inner(
|
||||
struct ConnectionContext {
|
||||
http_server: Arc<ConnBuilder<HttpTaskExecutor>>,
|
||||
task_owner: HttpTaskExecutor,
|
||||
s3_service: S3EdgeService,
|
||||
s3_service: S3Service,
|
||||
compression_config: HttpCompressionConfig,
|
||||
is_console: bool,
|
||||
/// Whether `RUSTFS_SERVER_DOMAINS` is configured (i.e. s3s virtual-hosted-style routing is active).
|
||||
|
||||
@@ -16,10 +16,6 @@ mod audit;
|
||||
mod compress;
|
||||
pub mod cors;
|
||||
mod event;
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
mod gateway_handlers;
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
mod gateway_stack;
|
||||
mod health;
|
||||
mod http;
|
||||
mod hybrid;
|
||||
@@ -33,7 +29,6 @@ mod runtime;
|
||||
pub(crate) mod runtime_sources;
|
||||
mod service_state;
|
||||
mod ssec_transport;
|
||||
mod stack;
|
||||
pub mod tls_material;
|
||||
pub(crate) mod tls_publish;
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ fn carries_ssec_headers(headers: &HeaderMap) -> bool {
|
||||
/// protocol is only consulted as a second source because the trusted-proxy
|
||||
/// layer has already decided whether the peer is allowed to assert it; a
|
||||
/// request that arrives direct carries no such assertion.
|
||||
pub(crate) fn is_secure_transport(connection_is_tls: bool, client_info: Option<&ClientInfo>) -> bool {
|
||||
fn is_secure_transport(connection_is_tls: bool, client_info: Option<&ClientInfo>) -> bool {
|
||||
if connection_is_tls {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -1,253 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The S3 protocol stack one server runs, selected once at startup (rustfs/backlog#2734 T2.5).
|
||||
//!
|
||||
//! Responsible for: [`S3StackService`], the service mounted where the legacy S3 service used to
|
||||
//! be, and [`select_stack`], the only place a stack is chosen. The whole stack is one or the
|
||||
//! other: no request is routed between stacks by its operation, and a request the gateway stack
|
||||
//! refuses is answered by that refusal and never retried on the legacy stack.
|
||||
//! NOT responsible for: assembling either stack (`server/http.rs` builds the legacy service,
|
||||
//! `server/gateway_stack.rs` the gateway one), or the middleware around them.
|
||||
//! Upstream: `server/http.rs`. Downstream: the legacy S3 service and the gateway stack.
|
||||
//!
|
||||
//! This file names neither stack's types: it is generic over two services with one response
|
||||
//! type, so `server/http.rs` decides what they are and how their bodies meet.
|
||||
|
||||
use super::layer::is_sts_query_request;
|
||||
use super::prefix::{HEALTH_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, has_path_prefix, is_admin_path};
|
||||
use crate::admin::console::is_console_path;
|
||||
use crate::config::S3Stack;
|
||||
use http::{HeaderMap, Method, Request, Uri};
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll, ready};
|
||||
use tower::Service;
|
||||
|
||||
/// The MinIO-compatible health probes share this prefix.
|
||||
const MINIO_HEALTH_PREFIX: &str = "/minio/health";
|
||||
|
||||
/// The S3 service of one server: the legacy stack, or the gateway stack with the legacy router
|
||||
/// kept for the routes that are not S3 operations.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct S3StackService<L, G> {
|
||||
legacy: L,
|
||||
gateway: Option<G>,
|
||||
}
|
||||
|
||||
impl<L, G> S3StackService<L, G> {
|
||||
/// The stack this service runs.
|
||||
pub(crate) fn selected(&self) -> S3Stack {
|
||||
match self.gateway {
|
||||
Some(_) => S3Stack::Gateway,
|
||||
None => S3Stack::Legacy,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the service for the selected stack.
|
||||
///
|
||||
/// `assemble_gateway` runs only when the gateway stack is selected, and its failure is the
|
||||
/// failure of the whole selection: the caller refuses to start.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Whatever `assemble_gateway` refused with. The legacy stack is never substituted.
|
||||
pub(crate) fn select_stack<L, G, E>(
|
||||
stack: S3Stack,
|
||||
legacy: L,
|
||||
assemble_gateway: impl FnOnce() -> Result<G, E>,
|
||||
) -> Result<S3StackService<L, G>, E> {
|
||||
let gateway = match stack {
|
||||
S3Stack::Legacy => None,
|
||||
S3Stack::Gateway => Some(assemble_gateway()?),
|
||||
};
|
||||
Ok(S3StackService { legacy, gateway })
|
||||
}
|
||||
|
||||
/// Whether the legacy router answers this request while the gateway stack is selected: the admin
|
||||
/// and table-catalog APIs, the console, STS, health and profiling, none of which is an S3
|
||||
/// operation. Every other request is the gateway's, whatever it answers.
|
||||
fn is_legacy_only_route(method: &Method, uri: &Uri, headers: &HeaderMap) -> bool {
|
||||
let path = uri.path();
|
||||
is_admin_path(path)
|
||||
|| is_console_path(path)
|
||||
|| has_path_prefix(path, HEALTH_PREFIX)
|
||||
|| has_path_prefix(path, MINIO_HEALTH_PREFIX)
|
||||
|| path == PROFILE_CPU_PATH
|
||||
|| path == PROFILE_MEMORY_PATH
|
||||
|| is_sts_query_request(method, uri, headers)
|
||||
}
|
||||
|
||||
impl<L, G, B, R, E> Service<Request<B>> for S3StackService<L, G>
|
||||
where
|
||||
L: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
G: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
L::Future: Send + 'static,
|
||||
G::Future: Send + 'static,
|
||||
{
|
||||
type Response = http::Response<R>;
|
||||
type Error = E;
|
||||
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
if let Some(gateway) = &mut self.gateway {
|
||||
ready!(gateway.poll_ready(cx))?;
|
||||
}
|
||||
self.legacy.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, request: Request<B>) -> Self::Future {
|
||||
match &mut self.gateway {
|
||||
// REMOVED BY T2.11: the gateway serves these routes itself once they are migrated.
|
||||
Some(gateway) if !is_legacy_only_route(request.method(), request.uri(), request.headers()) => {
|
||||
Box::pin(gateway.call(request))
|
||||
}
|
||||
_ => Box::pin(self.legacy.call(request)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::{Response, StatusCode};
|
||||
use std::convert::Infallible;
|
||||
use std::future::{Ready, ready as ready_future};
|
||||
|
||||
/// Answers every request with its own name.
|
||||
#[derive(Clone)]
|
||||
struct Named(&'static str);
|
||||
|
||||
impl Service<Request<()>> for Named {
|
||||
type Response = Response<&'static str>;
|
||||
type Error = Infallible;
|
||||
type Future = Ready<Result<Self::Response, Self::Error>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, _request: Request<()>) -> Self::Future {
|
||||
ready_future(Ok(Response::new(self.0)))
|
||||
}
|
||||
}
|
||||
|
||||
fn request(method: Method, target: &str) -> Request<()> {
|
||||
Request::builder()
|
||||
.method(method)
|
||||
.uri(target)
|
||||
.body(())
|
||||
.expect("static request")
|
||||
}
|
||||
|
||||
async fn answered_by(service: &mut S3StackService<Named, Named>, request: Request<()>) -> &'static str {
|
||||
let response = service.call(request).await.expect("infallible");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
response.into_body()
|
||||
}
|
||||
|
||||
fn gateway_stack() -> S3StackService<Named, Named> {
|
||||
select_stack(S3Stack::Gateway, Named("legacy"), || Ok::<_, Infallible>(Named("gateway"))).expect("assembles")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_gateway_assembly_failure_fails_the_selection_instead_of_falling_back() {
|
||||
let selection = select_stack::<_, Named, _>(S3Stack::Gateway, Named("legacy"), || Err("the assembly was refused"));
|
||||
|
||||
assert_eq!(selection.err(), Some("the assembly was refused"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_selection_never_assembles_a_gateway() {
|
||||
let service = select_stack::<_, Named, &str>(S3Stack::Legacy, Named("legacy"), || {
|
||||
panic!("the legacy stack must not assemble the gateway")
|
||||
})
|
||||
.expect("legacy needs no assembly");
|
||||
|
||||
assert_eq!(service.selected(), S3Stack::Legacy);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_legacy_stack_answers_every_request_itself() {
|
||||
let mut service = select_stack(S3Stack::Legacy, Named("legacy"), || Ok::<_, Infallible>(Named("gateway")))
|
||||
.expect("legacy needs no assembly");
|
||||
|
||||
for target in ["/", "/bucket/key", "/rustfs/admin/v3/info", "/health"] {
|
||||
assert_eq!(answered_by(&mut service, request(Method::GET, target)).await, "legacy", "{target}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_gateway_stack_answers_every_s3_request_whatever_the_operation() {
|
||||
let mut service = gateway_stack();
|
||||
assert_eq!(service.selected(), S3Stack::Gateway);
|
||||
|
||||
for (method, target) in [
|
||||
(Method::GET, "/"),
|
||||
(Method::GET, "/bucket/key"),
|
||||
(Method::PUT, "/bucket/key"),
|
||||
(Method::DELETE, "/bucket"),
|
||||
(Method::GET, "/bucket?versions"),
|
||||
(Method::POST, "/bucket/key?uploads"),
|
||||
// Prefixes of the legacy-only routes that are bucket names, not those routes.
|
||||
(Method::GET, "/healthy/key"),
|
||||
(Method::GET, "/rustfs/administrator"),
|
||||
(Method::GET, "/minio/healthcheck"),
|
||||
(Method::GET, "/profile/cpus"),
|
||||
] {
|
||||
assert_eq!(
|
||||
answered_by(&mut service, request(method.clone(), target)).await,
|
||||
"gateway",
|
||||
"{method} {target}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_gateway_stack_leaves_the_non_s3_routes_to_the_legacy_router() {
|
||||
let mut service = gateway_stack();
|
||||
|
||||
for target in [
|
||||
"/rustfs/admin/v3/info",
|
||||
"/minio/admin/v3/info",
|
||||
"/iceberg/v1/config",
|
||||
"/health",
|
||||
"/health/ready",
|
||||
"/minio/health/live",
|
||||
"/profile/cpu",
|
||||
"/profile/memory",
|
||||
] {
|
||||
assert_eq!(answered_by(&mut service, request(Method::GET, target)).await, "legacy", "{target}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_sts_form_post_stays_on_the_legacy_router_and_a_plain_post_does_not() {
|
||||
let mut service = gateway_stack();
|
||||
let sts = Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri("/")
|
||||
.header(http::header::CONTENT_TYPE, "application/x-www-form-urlencoded")
|
||||
.body(())
|
||||
.expect("static request");
|
||||
|
||||
assert!(
|
||||
is_sts_query_request(sts.method(), sts.uri(), sts.headers()),
|
||||
"the fixture is an STS request"
|
||||
);
|
||||
assert_eq!(answered_by(&mut service, sts).await, "legacy");
|
||||
assert_eq!(answered_by(&mut service, request(Method::POST, "/")).await, "gateway");
|
||||
}
|
||||
}
|
||||
@@ -1334,11 +1334,11 @@ fn test_delete_bucket_broadcast_fences_target_membership_through_delivery() {
|
||||
"a failed destructive delivery must remain visible for operator repair"
|
||||
);
|
||||
|
||||
let usecase = include_str!("../app/bucket_usecase.rs");
|
||||
let usecase = crate::guard_sources::SourceSet::read("src/app/bucket_usecase.rs", "src/app/bucket_usecase");
|
||||
let delete = usecase
|
||||
.split("async fn execute_delete_bucket_inner")
|
||||
.nth(1)
|
||||
.and_then(|rest| rest.split("pub async fn execute_head_bucket").next())
|
||||
.after_unique_marker("async fn execute_delete_bucket_inner")
|
||||
.split("pub async fn execute_head_bucket")
|
||||
.next()
|
||||
.expect("delete bucket usecase");
|
||||
assert!(
|
||||
delete
|
||||
|
||||
@@ -41,8 +41,7 @@ const EVENT_SERVER_STARTING: &str = "server_starting";
|
||||
const EVENT_ACTION_CREDENTIALS_INITIALIZED: &str = "action_credentials_initialized";
|
||||
const EVENT_ACTION_CREDENTIALS_INITIALIZATION_FAILED: &str = "action_credentials_initialization_failed";
|
||||
const DEFAULT_CREDENTIALS_WARNING_MESSAGE: &str = "Detected default root credentials; set RUSTFS_ACCESS_KEY and RUSTFS_SECRET_KEY to non-default values for production deployments";
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
const GATEWAY_STACK_NOT_BUILT: &str = "RUSTFS_S3_STACK=gateway needs a binary built with the `gateway-stack` feature, which no release build enables (rustfs/backlog#2734)";
|
||||
const GATEWAY_STACK_NOT_ASSEMBLED: &str = "gateway stack is not assembled yet (rustfs/backlog#2734 T2.5)";
|
||||
|
||||
pub(crate) struct StartupListenContext {
|
||||
pub(crate) readiness: Arc<GlobalReadiness>,
|
||||
@@ -83,7 +82,12 @@ pub(crate) async fn init_startup_listen_context(
|
||||
config: &Config,
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
) -> Result<StartupListenContext> {
|
||||
refuse_unbuilt_stack(config.s3_stack)?;
|
||||
match config.s3_stack {
|
||||
S3Stack::Legacy => {}
|
||||
// REPLACED BY T2.5
|
||||
// Refuse before any listener or storage exists; never fall back to legacy.
|
||||
S3Stack::Gateway => return Err(Error::new(ErrorKind::Unsupported, GATEWAY_STACK_NOT_ASSEMBLED)),
|
||||
}
|
||||
crate::server::init_console_prefix()?;
|
||||
log_sanitized_server_config(config);
|
||||
let readiness = Arc::new(GlobalReadiness::new());
|
||||
@@ -134,18 +138,6 @@ pub(crate) async fn init_startup_listen_context(
|
||||
})
|
||||
}
|
||||
|
||||
/// Refuses a selected S3 stack this binary cannot serve, before any listener or storage exists.
|
||||
/// The selection is never replaced by another stack.
|
||||
fn refuse_unbuilt_stack(stack: S3Stack) -> Result<()> {
|
||||
match stack {
|
||||
S3Stack::Legacy => Ok(()),
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
S3Stack::Gateway => Ok(()),
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
S3Stack::Gateway => Err(Error::new(ErrorKind::Unsupported, GATEWAY_STACK_NOT_BUILT)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_embedded_startup_config(
|
||||
address: String,
|
||||
access_key: String,
|
||||
@@ -174,10 +166,6 @@ pub(crate) async fn prepare_embedded_startup_config(
|
||||
config.secret_key = secret_key.clone();
|
||||
config.region = Some(region.clone());
|
||||
config.console_enable = false;
|
||||
// The stack is selected for the whole process, so an embedded server reads the same variable
|
||||
// the binary does.
|
||||
config.s3_stack = S3Stack::from_env()?;
|
||||
refuse_unbuilt_stack(config.s3_stack)?;
|
||||
|
||||
Ok(EmbeddedStartupConfig {
|
||||
config,
|
||||
@@ -530,26 +518,6 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
#[test]
|
||||
fn a_binary_without_the_gateway_stack_refuses_to_select_it() {
|
||||
let err = super::refuse_unbuilt_stack(crate::config::S3Stack::Gateway).expect_err("the gateway stack is not built");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::Unsupported);
|
||||
assert!(err.to_string().contains("gateway-stack"), "{err}");
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
#[test]
|
||||
fn a_binary_with_the_gateway_stack_accepts_its_selection() {
|
||||
super::refuse_unbuilt_stack(crate::config::S3Stack::Gateway).expect("the gateway stack is built");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_stack_is_always_selectable() {
|
||||
super::refuse_unbuilt_stack(crate::config::S3Stack::Legacy).expect("legacy is the default stack");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn init_startup_listen_context_rejects_console_s3_port_conflict() {
|
||||
let mut config = Config::new("127.0.0.1:9001", vec!["/tmp/rustfs-data".to_string()]);
|
||||
|
||||
+541
-398
File diff suppressed because it is too large
Load Diff
@@ -20,10 +20,8 @@
|
||||
//! counted and the count is carried on the next entry that is let through.
|
||||
|
||||
use crate::module_switches::is_audit_module_enabled;
|
||||
use crate::storage::helper::{
|
||||
AuditRequestView, DenialRequest, DenialScope, authorization_denial_entry, authorization_denial_entry_for_view,
|
||||
spawn_background_with_context,
|
||||
};
|
||||
use crate::storage::access::{ReqInfo, request_context_from_req};
|
||||
use crate::storage::helper::{DenialRequest, DenialScope, authorization_denial_entry, spawn_background_with_context};
|
||||
use crate::storage::request_context::RequestContext;
|
||||
use rustfs_audit::{AuditEntry, global::AuditLogger};
|
||||
use rustfs_policy::policy::action::Action;
|
||||
@@ -186,32 +184,21 @@ impl DenialAuditor {
|
||||
scope: DenialScope,
|
||||
now: Instant,
|
||||
) {
|
||||
self.audit_view_at(&AuditRequestView::legacy(req), now, || {
|
||||
authorization_denial_entry(req, action, reason, scope)
|
||||
});
|
||||
}
|
||||
|
||||
/// Audits a denial for a request an owned edge decoded into `view`.
|
||||
pub(crate) fn audit_owned(&self, view: &AuditRequestView<'_>, action: &Action, reason: &'static str, scope: DenialScope) {
|
||||
self.audit_view_at(view, (self.clock)(), || authorization_denial_entry_for_view(view, action, reason, scope));
|
||||
}
|
||||
|
||||
fn audit_view_at(&self, req: &AuditRequestView<'_>, now: Instant, build_entry: impl FnOnce() -> AuditEntry) {
|
||||
if !(self.enabled)() {
|
||||
return;
|
||||
}
|
||||
let anonymous = req.req_info.is_none_or(|info| info.cred.is_none());
|
||||
let anonymous = req.extensions.get::<ReqInfo>().is_none_or(|info| info.cred.is_none());
|
||||
let Admission::Emit { suppressed } = self.limiter.admit(anonymous, now) else {
|
||||
return;
|
||||
};
|
||||
let mut entry = build_entry();
|
||||
let mut entry = authorization_denial_entry(req, action, reason, scope);
|
||||
if suppressed > 0 {
|
||||
entry
|
||||
.tags
|
||||
.get_or_insert_with(Default::default)
|
||||
.insert(SUPPRESSED_DENIALS_TAG.to_string(), Value::from(suppressed));
|
||||
}
|
||||
(self.emit)(req.request_context(), entry);
|
||||
(self.emit)(request_context_from_req(req), entry);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+11
-25
@@ -45,7 +45,6 @@ use crate::storage::validate_website_configuration;
|
||||
use crate::table_catalog;
|
||||
use http::StatusCode;
|
||||
use metrics::{counter, histogram};
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_buckets as gateway_list_buckets;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
@@ -741,15 +740,6 @@ impl FS {
|
||||
}
|
||||
}
|
||||
|
||||
/// A ListBuckets value the s3s edge and the gateway's types disagree on. The seam converts every
|
||||
/// member both sides hold, so this is a defect in the conversion, never the caller's fault.
|
||||
fn list_buckets_seam_error(error: rustfs_gateway_types::compat::ConversionError) -> S3Error {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("failed to convert ListBuckets through the gateway compatibility seam: {error}"),
|
||||
)
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl S3 for FS {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
@@ -1606,17 +1596,7 @@ impl S3 for FS {
|
||||
// List buckets not associated with a bucket, give it bucket label "*" to denote "all".
|
||||
record_s3_op(S3Operation::ListBuckets);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
// The use case speaks the envelope and the gateway's types; this edge only converts.
|
||||
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
|
||||
let input = gateway_list_buckets::input_from_s3s(req.input).map_err(list_buckets_seam_error)?;
|
||||
let output = usecase.execute_list_buckets(envelope, input).await?;
|
||||
let mut output = gateway_list_buckets::output_to_s3s(output).map_err(list_buckets_seam_error)?;
|
||||
// The seam writes a listing without buckets as an absent member. This edge has always
|
||||
// answered the element, empty or not, and clients index the answer by it.
|
||||
if output.buckets.is_none() {
|
||||
output.buckets = Some(Vec::new());
|
||||
}
|
||||
Ok(S3Response::new(output))
|
||||
usecase.execute_list_buckets(req).await
|
||||
}
|
||||
|
||||
async fn list_multipart_uploads(
|
||||
@@ -2296,13 +2276,13 @@ impl S3 for FS {
|
||||
|
||||
/// Every mutating S3 operation that has no bucket notification event reaches the audit
|
||||
/// stream only through `audit_only_operation`; a new such operation, or a dropped
|
||||
/// wrapper, must fail here rather than silently go unaudited.
|
||||
/// wrapper, must fail here rather than silently go unaudited. The wrappers are looked
|
||||
/// up in this file and in `legacy_edge/`, where a split of this file moves them.
|
||||
#[cfg(test)]
|
||||
mod audit_coverage_tests {
|
||||
use crate::guard_sources::SourceSet;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
|
||||
const SOURCE: &str = include_str!("ecfs.rs");
|
||||
|
||||
fn is_read_only(op: S3Operation) -> bool {
|
||||
let name = op.as_str().trim_start_matches("s3:");
|
||||
["Get", "Head", "List", "Select"]
|
||||
@@ -2320,9 +2300,15 @@ mod audit_coverage_tests {
|
||||
expected.push(S3Operation::AbortMultipartUpload);
|
||||
assert_eq!(expected.len(), 22, "an operation was added or removed: {expected:?}");
|
||||
|
||||
let source = SourceSet::read("src/storage/ecfs.rs", "src/storage/legacy_edge");
|
||||
for op in expected {
|
||||
let wrapper = format!("audit_only_operation(req, S3Operation::{op:?},");
|
||||
assert!(SOURCE.contains(&wrapper), "{} is not run through audit_only_operation", op.as_str());
|
||||
assert!(
|
||||
source.contains(&wrapper),
|
||||
"{} is not run through audit_only_operation in {}",
|
||||
op.as_str(),
|
||||
source.describe()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::storage::request_context::RequestContext;
|
||||
use crate::storage::sse::KmsRequestAuditScope;
|
||||
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
|
||||
use hashbrown::HashMap;
|
||||
use http::{HeaderMap, StatusCode, Uri};
|
||||
use http::StatusCode;
|
||||
use metrics::counter;
|
||||
use rustfs_audit::{
|
||||
ApiDetails, AuditEntry, ObjectVersion,
|
||||
@@ -38,64 +38,12 @@ use rustfs_utils::http::headers::AMZ_REQUEST_ID;
|
||||
use s3s::{S3Request, S3Response, S3Result};
|
||||
use serde_json::Value;
|
||||
use std::future::Future;
|
||||
use std::net::SocketAddr;
|
||||
use tokio::runtime::{Builder, Handle};
|
||||
|
||||
/// Request type seen by the denial auditor, so `denial_audit` stays off the direct s3s surface.
|
||||
pub(crate) type DenialRequest<T> = S3Request<T>;
|
||||
use tracing::{Instrument, info_span, warn};
|
||||
|
||||
/// The request fields used to build an audit entry, shared by both request edges.
|
||||
pub(crate) struct AuditRequestView<'a> {
|
||||
headers: &'a HeaderMap,
|
||||
uri: &'a Uri,
|
||||
pub(crate) req_info: Option<&'a ReqInfo>,
|
||||
client_info: Option<&'a ClientInfo>,
|
||||
remote_addr: Option<SocketAddr>,
|
||||
request_context: Option<&'a RequestContext>,
|
||||
}
|
||||
|
||||
impl<'a> AuditRequestView<'a> {
|
||||
pub(crate) fn legacy<T>(req: &'a S3Request<T>) -> Self {
|
||||
Self {
|
||||
headers: &req.headers,
|
||||
uri: &req.uri,
|
||||
req_info: req.extensions.get::<ReqInfo>(),
|
||||
client_info: req.extensions.get::<ClientInfo>(),
|
||||
remote_addr: req
|
||||
.extensions
|
||||
.get::<Option<RemoteAddr>>()
|
||||
.and_then(|addr| addr.map(|addr| addr.0)),
|
||||
request_context: req.extensions.get::<RequestContext>(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The view of a request whose facts an owned edge already decoded.
|
||||
pub(crate) fn from_facts(
|
||||
headers: &'a HeaderMap,
|
||||
uri: &'a Uri,
|
||||
req_info: Option<&'a ReqInfo>,
|
||||
client_info: Option<&'a ClientInfo>,
|
||||
remote_addr: Option<SocketAddr>,
|
||||
request_context: Option<&'a RequestContext>,
|
||||
) -> Self {
|
||||
Self {
|
||||
headers,
|
||||
uri,
|
||||
req_info,
|
||||
client_info,
|
||||
remote_addr,
|
||||
request_context,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn request_context(&self) -> Option<RequestContext> {
|
||||
self.request_context
|
||||
.cloned()
|
||||
.or_else(|| self.req_info.and_then(|info| info.request_context.clone()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
use super::storage_api::ecstore_diagnostic;
|
||||
use crate::storage::storage_api::helper_consumer::StorageObjectInfo as ObjectInfo;
|
||||
@@ -179,27 +127,32 @@ pub(crate) fn build_event_resp_elements<T>(response: &S3Response<T>, request_id:
|
||||
|
||||
/// Builds the audit entry shared by every request-scoped audit source: caller identity,
|
||||
/// client address and the request line, without any outcome.
|
||||
fn request_audit_builder(
|
||||
req: &AuditRequestView<'_>,
|
||||
fn request_audit_builder<T>(
|
||||
req: &S3Request<T>,
|
||||
req_info: Option<&ReqInfo>,
|
||||
event: EventName,
|
||||
trigger: &str,
|
||||
api: ApiDetails,
|
||||
request_id: &str,
|
||||
) -> AuditEntryBuilder {
|
||||
let remote_host = req
|
||||
.client_info
|
||||
.extensions
|
||||
.get::<ClientInfo>()
|
||||
.map(|info| info.real_ip.to_string())
|
||||
.or_else(|| req.remote_addr.map(|addr| addr.ip().to_string()))
|
||||
.or_else(|| {
|
||||
req.extensions
|
||||
.get::<Option<RemoteAddr>>()
|
||||
.and_then(|addr| addr.map(|addr| addr.0.ip().to_string()))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let mut builder = AuditEntryBuilder::new("1.0", event, trigger, api)
|
||||
.remote_host(remote_host)
|
||||
.user_agent(get_request_user_agent(req.headers))
|
||||
.req_host(get_request_host(req.headers))
|
||||
.user_agent(get_request_user_agent(&req.headers))
|
||||
.req_host(get_request_host(&req.headers))
|
||||
.req_path(req.uri.path().to_string())
|
||||
.req_query(extract_params_header(req.headers))
|
||||
.req_query(extract_params_header(&req.headers))
|
||||
.request_id(request_id);
|
||||
if let Some(cred) = req
|
||||
.req_info
|
||||
if let Some(cred) = req_info
|
||||
.and_then(|info| info.cred.as_ref())
|
||||
.filter(|cred| !cred.access_key.is_empty())
|
||||
{
|
||||
@@ -421,9 +374,8 @@ impl OperationHelper {
|
||||
let request_context = request_context.unwrap_or_else(|| RequestContext::from_external_headers(&req.headers));
|
||||
let request_id = request_context.request_id.clone();
|
||||
|
||||
let audit_builder = audit_enabled.then(|| {
|
||||
request_audit_builder(&AuditRequestView::legacy(req), event, trigger, api_builder.clone().build(), &request_id)
|
||||
});
|
||||
let audit_builder =
|
||||
audit_enabled.then(|| request_audit_builder(req, req_info, event, trigger, api_builder.clone().build(), &request_id));
|
||||
|
||||
let event_object = ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
@@ -650,16 +602,7 @@ pub(crate) fn authorization_denial_entry<T>(
|
||||
reason: &'static str,
|
||||
scope: DenialScope,
|
||||
) -> AuditEntry {
|
||||
authorization_denial_entry_for_view(&AuditRequestView::legacy(req), action, reason, scope)
|
||||
}
|
||||
|
||||
pub(crate) fn authorization_denial_entry_for_view(
|
||||
req: &AuditRequestView<'_>,
|
||||
action: &Action,
|
||||
reason: &'static str,
|
||||
scope: DenialScope,
|
||||
) -> AuditEntry {
|
||||
let req_info = req.req_info;
|
||||
let req_info = req.extensions.get::<ReqInfo>();
|
||||
let path = req.uri.path().trim_start_matches('/');
|
||||
let mut segs = path.splitn(2, '/');
|
||||
let path_bucket = segs.next().unwrap_or("");
|
||||
@@ -688,18 +631,23 @@ pub(crate) fn authorization_denial_entry_for_view(
|
||||
api = api.object(object);
|
||||
}
|
||||
|
||||
let request_context = req
|
||||
.request_context()
|
||||
.unwrap_or_else(|| RequestContext::from_external_headers(req.headers));
|
||||
let request_context = request_context_from_req(req).unwrap_or_else(|| RequestContext::from_external_headers(&req.headers));
|
||||
let mut tags = HashMap::new();
|
||||
tags.insert("denialReason".to_string(), Value::String(reason.to_string()));
|
||||
if let DenialScope::Keys(denied) = scope {
|
||||
tags.insert("deniedKeys".to_string(), Value::from(denied));
|
||||
}
|
||||
request_audit_builder(req, EventName::AuditS3Operation, action_name, api.build(), &request_context.request_id)
|
||||
.error("Access Denied")
|
||||
.tags(tags)
|
||||
.build()
|
||||
request_audit_builder(
|
||||
req,
|
||||
req_info,
|
||||
EventName::AuditS3Operation,
|
||||
action_name,
|
||||
api.build(),
|
||||
&request_context.request_id,
|
||||
)
|
||||
.error("Access Denied")
|
||||
.tags(tags)
|
||||
.build()
|
||||
}
|
||||
|
||||
/// Runs `handler` on `req` and records the outcome as an audit-only entry for `op`.
|
||||
@@ -1487,7 +1435,8 @@ mod tests {
|
||||
..Default::default()
|
||||
});
|
||||
let entry = super::request_audit_builder(
|
||||
&super::AuditRequestView::legacy(&req),
|
||||
&req,
|
||||
req.extensions.get::<ReqInfo>(),
|
||||
EventName::ObjectAccessedGet,
|
||||
"GetObject",
|
||||
rustfs_audit::ApiDetails::default(),
|
||||
|
||||
@@ -13,14 +13,15 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::s3_api::common::rustfs_owner;
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::contract::list::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::contract::{
|
||||
bucket::BucketInfo,
|
||||
list::{ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info},
|
||||
};
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::to_s3s_xml_etag;
|
||||
use percent_encoding::percent_decode_str;
|
||||
use s3s::dto::{
|
||||
CommonPrefix, DeleteMarkerEntry, EncodingType, ListObjectVersionsOutput, ListObjectsOutput, ListObjectsV2Output, Object,
|
||||
ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Timestamp,
|
||||
Bucket, CommonPrefix, DeleteMarkerEntry, EncodingType, ListBucketsOutput, ListObjectVersionsOutput, ListObjectsOutput,
|
||||
ListObjectsV2Output, Object, ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Timestamp,
|
||||
};
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
use tracing::debug;
|
||||
@@ -125,6 +126,23 @@ pub(crate) fn build_list_objects_output(v2: ListObjectsV2Output, request_marker:
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_list_buckets_output(bucket_infos: &[BucketInfo]) -> ListBucketsOutput {
|
||||
let buckets: Vec<Bucket> = bucket_infos
|
||||
.iter()
|
||||
.map(|bucket_info| Bucket {
|
||||
creation_date: Some(Timestamp::from(bucket_info.created.unwrap_or(time::OffsetDateTime::UNIX_EPOCH))),
|
||||
name: Some(bucket_info.name.clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
|
||||
ListBucketsOutput {
|
||||
buckets: Some(buckets),
|
||||
owner: Some(rustfs_owner()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_list_object_versions_params(
|
||||
prefix: Option<String>,
|
||||
delimiter: Option<String>,
|
||||
@@ -432,16 +450,19 @@ fn calculate_next_marker(v2: &ListObjectsV2Output) -> Option<String> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
ListObjectVersionsInfo, ListObjectVersionsParams, ListObjectsV2Info, build_list_object_versions_output,
|
||||
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
|
||||
parse_list_object_versions_params, parse_list_objects_v2_params, to_s3s_xml_etag,
|
||||
ListObjectVersionsInfo, ListObjectVersionsParams, ListObjectsV2Info, build_list_buckets_output,
|
||||
build_list_object_versions_output, build_list_objects_output, build_list_objects_v2_output,
|
||||
encode_list_objects_v2_continuation_token, parse_list_object_versions_params, parse_list_objects_v2_params,
|
||||
to_s3s_xml_etag,
|
||||
};
|
||||
use crate::storage::s3_api::common::rustfs_owner;
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::StorageObjectInfo as ObjectInfo;
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::contract::bucket::BucketInfo;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::{CommonPrefix, EncodingType, ListObjectsV2Output, Object};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
@@ -504,6 +525,36 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_list_buckets_output_maps_bucket_infos_and_owner() {
|
||||
let bucket_infos = vec![
|
||||
BucketInfo {
|
||||
name: "bucket-a".to_string(),
|
||||
created: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
},
|
||||
BucketInfo {
|
||||
name: "bucket-b".to_string(),
|
||||
created: None,
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
|
||||
let output = build_list_buckets_output(&bucket_infos);
|
||||
let buckets = output.buckets.as_ref().expect("buckets should be present");
|
||||
let owner = output.owner.as_ref().expect("owner should be present");
|
||||
|
||||
assert_eq!(buckets.len(), 2);
|
||||
assert_eq!(buckets[0].name.as_deref(), Some("bucket-a"));
|
||||
assert_eq!(buckets[0].creation_date, Some(s3s::dto::Timestamp::from(OffsetDateTime::UNIX_EPOCH)));
|
||||
assert_eq!(buckets[1].name.as_deref(), Some("bucket-b"));
|
||||
assert_eq!(buckets[1].creation_date, Some(s3s::dto::Timestamp::from(OffsetDateTime::UNIX_EPOCH)));
|
||||
|
||||
let expected_owner = rustfs_owner();
|
||||
assert_eq!(owner.display_name, expected_owner.display_name);
|
||||
assert_eq!(owner.id, expected_owner.id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_objects_marker_echoes_request_value() {
|
||||
let output = build_list_objects_output(ListObjectsV2Output::default(), Some("m-1".to_string()));
|
||||
|
||||
@@ -25,14 +25,11 @@ impl RequestEnvelope {
|
||||
/// Copies every request fact of `req` the app layer reads.
|
||||
pub fn from_s3s<T>(req: &S3Request<T>) -> Self {
|
||||
let credentials = req.credentials.as_ref().map(|credentials| {
|
||||
let session_token = get_session_token(&req.uri, &req.headers).map(str::to_owned);
|
||||
// An empty secret is not a key: carry it as absent so every reader fails closed.
|
||||
RequestCredentials::new(
|
||||
credentials.access_key.clone(),
|
||||
credentials.secret_key.expose().to_owned(),
|
||||
session_token.clone(),
|
||||
get_session_token(&req.uri, &req.headers).map(str::to_owned),
|
||||
)
|
||||
.unwrap_or_else(|_| RequestCredentials::without_secret_key(credentials.access_key.clone(), session_token))
|
||||
});
|
||||
let head = RequestHead {
|
||||
method: req.method.clone(),
|
||||
@@ -181,7 +178,7 @@ mod tests {
|
||||
assert_eq!(envelope.headers(), &req.headers);
|
||||
let credentials = envelope.credentials().expect("credentials");
|
||||
assert_eq!(credentials.access_key(), "AKIAENVELOPE");
|
||||
assert_eq!(credentials.secret_key(), Some("envelope-secret"));
|
||||
assert_eq!(credentials.secret_key(), "envelope-secret");
|
||||
assert_eq!(credentials.session_token(), Some("header-token"));
|
||||
assert_eq!(envelope.region(), Some("eu-west-3"));
|
||||
assert_eq!(envelope.service(), Some("s3"));
|
||||
|
||||
@@ -147,36 +147,17 @@ pub(crate) use super::sse::{
|
||||
|
||||
pub(crate) mod access_consumer {
|
||||
pub(crate) use super::super::access::{
|
||||
AuthorizationRequest, BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard,
|
||||
ListBucketsAuthzRequest, MultipartStagingOperation, ObjectTagConditions, OdmReadGenerationGuard,
|
||||
PendingDeleteBucketGenerationGuard, PostObjectPolicyHeaders, PostObjectRequestMarker, ReqInfo, RequestedCopySource,
|
||||
RequestedObjectLegalHold, RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags,
|
||||
StagedMultipartPartAuthorization, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess,
|
||||
TableDataPlaneListCursorPosition, TableDataPlanePublicationGuards, apply_bucket_generation_guard,
|
||||
apply_copy_source_bucket_generation_guard, authorize_copy_object_tags, authorize_request, authorize_request_core,
|
||||
bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header,
|
||||
load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
|
||||
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller,
|
||||
replication_proxy_allowed, replication_request_authorized, req_info_mut, req_info_ref,
|
||||
retained_table_data_plane_publication_guards, set_requested_object_lock_retain_until,
|
||||
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, MultipartStagingOperation,
|
||||
ObjectTagConditions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectRequestMarker, ReqInfo,
|
||||
TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, TableDataPlaneListCursorPosition,
|
||||
TableDataPlanePublicationGuards, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard,
|
||||
authorize_copy_object_tags, authorize_internal_object_request, authorize_request, bucket_config_mutation_incarnation,
|
||||
delete_object_authorize_action, has_bypass_governance_header, load_bucket_generation_from_store,
|
||||
log_list_buckets_iam_implicit_deny, odm_read_generation, prepare_list_buckets_iam_authorization,
|
||||
prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller, replication_proxy_allowed,
|
||||
replication_request_authorized, req_info_mut, req_info_ref, retained_table_data_plane_publication_guards,
|
||||
set_requested_object_lock_retain_until,
|
||||
};
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) use super::super::access::{admit_operation, resolve_req_info};
|
||||
pub(crate) use super::super::denial_audit::DenialAuditor;
|
||||
pub(crate) use super::super::helper::{AuditRequestView, DenialScope};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod owned_parity_consumer {
|
||||
pub(crate) use super::super::access::{
|
||||
ObjectTagConditions, PostObjectPolicyHeaders, PostObjectRequestMarker, ReqInfo, RequestedCopySource,
|
||||
RequestedObjectLegalHold, RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags,
|
||||
TableDataPlanePublicationGuards, authorize_request, parity_test_support,
|
||||
};
|
||||
pub(crate) use super::super::denial_audit::{DenialAuditor, DenialRateLimiter, SUPPRESSED_DENIALS_TAG};
|
||||
pub(crate) use super::super::ecfs::FS;
|
||||
pub(crate) use super::super::helper::{DenialScope, authorization_denial_entry, authorization_denial_entry_for_view};
|
||||
pub(crate) use super::super::request_context::RequestContext;
|
||||
}
|
||||
|
||||
pub(crate) mod concurrency_consumer {
|
||||
@@ -371,12 +352,16 @@ pub(crate) mod heal_control_startup_consumer {
|
||||
pub(crate) mod s3_api_consumer {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use super::super::super::s3_api::bucket::{
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_object_versions_output, build_list_objects_output,
|
||||
build_list_objects_v2_output, encode_list_objects_v2_continuation_token, parse_list_object_versions_params,
|
||||
parse_list_objects_v2_params,
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
|
||||
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
|
||||
parse_list_object_versions_params, parse_list_objects_v2_params,
|
||||
};
|
||||
|
||||
pub(crate) mod contract {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use super::super::super::super::contract::bucket::BucketInfo;
|
||||
}
|
||||
|
||||
pub(crate) mod list {
|
||||
pub(crate) use super::super::super::super::contract::list::{ListObjectVersionsInfo, ListObjectsV2Info};
|
||||
}
|
||||
|
||||
@@ -231,13 +231,6 @@ pub(crate) mod server {
|
||||
pub(crate) use crate::storage::storage_api::request_context_consumer::RequestContext;
|
||||
}
|
||||
|
||||
/// The access-phase functions the gateway stack's handlers run, shared with the legacy
|
||||
/// access hooks.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) mod access {
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{admit_operation, resolve_req_info};
|
||||
}
|
||||
|
||||
pub(crate) mod rpc {
|
||||
pub(crate) use crate::storage::storage_api::rpc_consumer::InternodeRpcService;
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -14478,31 +14478,31 @@ fn object_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
|
||||
#[test]
|
||||
fn multipart_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
let source = include_str!("../app/multipart_usecase.rs");
|
||||
let source = crate::guard_sources::SourceSet::read("src/app/multipart_usecase.rs", "src/app/multipart_usecase");
|
||||
|
||||
assert_eq!(
|
||||
source
|
||||
.matches("validate_table_catalog_object_mutation(&bucket, &key).await?;")
|
||||
.count(),
|
||||
4
|
||||
source.count("validate_table_catalog_object_mutation(&bucket, &key).await?;"),
|
||||
4,
|
||||
"reserved-prefix guard calls across {}",
|
||||
source.describe()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_metadata_mutation_entrypoints_call_reserved_prefix_guard() {
|
||||
let source = include_str!("../storage/ecfs.rs");
|
||||
let source = crate::guard_sources::SourceSet::read("src/storage/ecfs.rs", "src/storage/legacy_edge");
|
||||
|
||||
assert_eq!(
|
||||
source
|
||||
.matches("validate_table_catalog_object_mutation(&bucket, &object).await?;")
|
||||
.count(),
|
||||
2
|
||||
source.count("validate_table_catalog_object_mutation(&bucket, &object).await?;"),
|
||||
2,
|
||||
"reserved-prefix guard calls across {}",
|
||||
source.describe()
|
||||
);
|
||||
assert_eq!(
|
||||
source
|
||||
.matches("validate_table_catalog_object_mutation(&bucket, &key).await?;")
|
||||
.count(),
|
||||
2
|
||||
source.count("validate_table_catalog_object_mutation(&bucket, &key).await?;"),
|
||||
2,
|
||||
"reserved-prefix guard calls across {}",
|
||||
source.describe()
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,402 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The gateway S3 stack inside an embedded RustFS server (rustfs/backlog#2734 T2.5): one
|
||||
// operation, ListBuckets, is served end to end, and every other S3 operation is answered
|
||||
// `501 NotImplemented`. Requests are signed by the AWS SDK and travel through the whole
|
||||
// middleware stack, so what is asserted here is what a client sees.
|
||||
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use rustfs::app::context::get_global_app_context;
|
||||
use rustfs::embedded::{RustFSServer, RustFSServerBuilder, find_available_port};
|
||||
use rustfs_io_metrics::s3_http_metrics::s3_http_metrics_snapshot;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_storage_api::{BucketOperations, MakeBucketOptions};
|
||||
use std::collections::BTreeMap;
|
||||
use std::future::Future;
|
||||
|
||||
mod common;
|
||||
|
||||
const ROOT_ACCESS_KEY: &str = "gatewaystackroot";
|
||||
const ROOT_SECRET_KEY: &str = "gateway-stack-root-secret";
|
||||
const USER_SECRET_KEY: &str = "gateway-stack-user-secret";
|
||||
const ENV_S3_STACK: &str = "RUSTFS_S3_STACK";
|
||||
const ENV_KEYSTONE_ENABLE: &str = "RUSTFS_KEYSTONE_ENABLE";
|
||||
const ENV_KEYSTONE_AUTH_URL: &str = "RUSTFS_KEYSTONE_AUTH_URL";
|
||||
/// A Keystone endpoint nothing listens on: these tests are about start-up, not about tokens.
|
||||
const UNREACHABLE_KEYSTONE: &str = "http://127.0.0.1:1/v3";
|
||||
|
||||
fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client {
|
||||
let config = Config::builder()
|
||||
.credentials_provider(Credentials::new(access_key, secret_key, None, None, "test"))
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(endpoint)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
}
|
||||
|
||||
/// Runs `test` against an embedded server started with `RUSTFS_S3_STACK` set to `stack`.
|
||||
fn with_stack<F, Fut>(stack: Option<&'static str>, test: F)
|
||||
where
|
||||
F: FnOnce(RustFSServer) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = ()> + 'static,
|
||||
{
|
||||
with_environment(vec![(ENV_S3_STACK, stack)], test);
|
||||
}
|
||||
|
||||
/// Runs `test` against an embedded server started under `environment`.
|
||||
fn with_environment<F, Fut>(environment: Vec<(&'static str, Option<&'static str>)>, test: F)
|
||||
where
|
||||
F: FnOnce(RustFSServer) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = ()> + 'static,
|
||||
{
|
||||
temp_env::with_vars(environment, || {
|
||||
common::run_embedded_test(move || async move {
|
||||
let Some(started) = start_server().await else { return };
|
||||
test(started.expect("start embedded server")).await;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/// Starts an embedded server on a free port; `None` when the sandbox allows no port at all.
|
||||
async fn start_server() -> Option<Result<RustFSServer, String>> {
|
||||
let port = match find_available_port() {
|
||||
Ok(port) => port,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
|
||||
Err(err) => panic!("find free port: {err}"),
|
||||
};
|
||||
let started = RustFSServerBuilder::new()
|
||||
.address(format!("127.0.0.1:{port}"))
|
||||
.access_key(ROOT_ACCESS_KEY)
|
||||
.secret_key(ROOT_SECRET_KEY)
|
||||
.build()
|
||||
.await;
|
||||
// Both renderings: the refusal's text may be in either.
|
||||
Some(started.map_err(|error| format!("{error} {error:?}")))
|
||||
}
|
||||
|
||||
/// The S3 error code and HTTP status an SDK call was refused with.
|
||||
fn refusal<E: ProvideErrorMetadata>(error: &SdkError<E>) -> (Option<String>, Option<u16>) {
|
||||
let status = error.raw_response().map(|response| response.status().as_u16());
|
||||
(error.code().map(str::to_owned), status)
|
||||
}
|
||||
|
||||
/// One series of the S3 request counter: method, operation and outcome.
|
||||
type Series = (&'static str, &'static str, &'static str);
|
||||
|
||||
/// How often each series of the S3 request counter has been counted in this process.
|
||||
fn s3_request_counts() -> BTreeMap<Series, u64> {
|
||||
s3_http_metrics_snapshot()
|
||||
.into_iter()
|
||||
.map(|series| ((series.method, series.operation, series.outcome), series.total))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Every series counted since `before`, with how often. Reading the whole counter, not one
|
||||
/// series, is what makes a second count under another label visible.
|
||||
fn counted_since(before: &BTreeMap<Series, u64>) -> BTreeMap<Series, u64> {
|
||||
s3_request_counts()
|
||||
.into_iter()
|
||||
.filter_map(|(series, total)| {
|
||||
let counted = total - before.get(&series).copied().unwrap_or(0);
|
||||
(counted > 0).then_some((series, counted))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn create_bucket_in_the_store(bucket: &str) {
|
||||
get_global_app_context()
|
||||
.expect("embedded server context")
|
||||
.object_store()
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket in the store");
|
||||
}
|
||||
|
||||
async fn create_user_without_a_policy(access_key: &str) {
|
||||
get_global_app_context()
|
||||
.expect("embedded server context")
|
||||
.iam()
|
||||
.handle()
|
||||
.create_user(
|
||||
access_key,
|
||||
&rustfs_madmin::AddOrUpdateUserReq {
|
||||
secret_key: USER_SECRET_KEY.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create user without a policy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_lists_the_caller_s_buckets_and_answers_every_other_operation_501() {
|
||||
with_stack(Some("gateway"), |server| async move {
|
||||
// CreateBucket is not served by this stack yet, so the buckets are made in the store.
|
||||
create_bucket_in_the_store("gateway-stack-alpha").await;
|
||||
create_bucket_in_the_store("gateway-stack-beta").await;
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
|
||||
let listing = client.list_buckets().send().await.expect("signed ListBuckets");
|
||||
let mut names: Vec<&str> = listing.buckets().iter().filter_map(|bucket| bucket.name()).collect();
|
||||
names.sort_unstable();
|
||||
assert_eq!(names, ["gateway-stack-alpha", "gateway-stack-beta"]);
|
||||
assert!(listing.owner().and_then(|owner| owner.id()).is_some(), "the listing names its owner");
|
||||
assert!(
|
||||
listing.buckets().iter().all(|bucket| bucket.creation_date().is_some()),
|
||||
"every bucket carries its creation date"
|
||||
);
|
||||
|
||||
let get = client
|
||||
.get_object()
|
||||
.bucket("gateway-stack-alpha")
|
||||
.key("object")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("GetObject is not registered");
|
||||
assert_eq!(refusal(&get), (Some("NotImplemented".to_owned()), Some(501)));
|
||||
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket("gateway-stack-alpha")
|
||||
.key("object")
|
||||
.body(ByteStream::from_static(b"payload"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("PutObject is not registered");
|
||||
assert_eq!(refusal(&put), (Some("NotImplemented".to_owned()), Some(501)));
|
||||
|
||||
let create = client
|
||||
.create_bucket()
|
||||
.bucket("gateway-stack-gamma")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("CreateBucket is not registered");
|
||||
assert_eq!(refusal(&create), (Some("NotImplemented".to_owned()), Some(501)));
|
||||
|
||||
let listing = client.list_buckets().send().await.expect("ListBuckets after the refusals");
|
||||
assert_eq!(listing.buckets().len(), 2, "a refused write created nothing");
|
||||
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_refuses_callers_it_cannot_admit() {
|
||||
with_stack(Some("gateway"), |server| async move {
|
||||
create_bucket_in_the_store("gateway-stack-alpha").await;
|
||||
create_user_without_a_policy("gatewaystacknopolicy").await;
|
||||
let endpoint = server.endpoint();
|
||||
|
||||
// Anonymous: no credentials at all.
|
||||
let anonymous = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("http client")
|
||||
.get(format!("{endpoint}/"))
|
||||
.send()
|
||||
.await
|
||||
.expect("anonymous ListBuckets is answered");
|
||||
assert_eq!(anonymous.status().as_u16(), 403);
|
||||
let request_id = anonymous
|
||||
.headers()
|
||||
.get("x-amz-request-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned)
|
||||
.expect("the refusal names its request");
|
||||
assert_eq!(
|
||||
anonymous.headers().get("x-request-id").and_then(|value| value.to_str().ok()),
|
||||
Some(request_id.as_str()),
|
||||
"one identifier names the request in both headers"
|
||||
);
|
||||
let body = anonymous.text().await.expect("refusal body");
|
||||
assert!(body.contains("<Code>AccessDenied</Code>"), "{body}");
|
||||
|
||||
// The right access key with the wrong secret.
|
||||
let wrong_secret = s3_client(&endpoint, ROOT_ACCESS_KEY, "not-the-root-secret")
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a wrong secret signs nothing");
|
||||
assert_eq!(refusal(&wrong_secret), (Some("SignatureDoesNotMatch".to_owned()), Some(403)));
|
||||
|
||||
// An access key nobody issued: the caller's fault, not the server's.
|
||||
let unknown_key = s3_client(&endpoint, "gatewaystackunknown", USER_SECRET_KEY)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an unknown access key is refused");
|
||||
assert_eq!(refusal(&unknown_key), (Some("InvalidAccessKeyId".to_owned()), Some(403)));
|
||||
|
||||
// A real user whose policy grants nothing: it may neither list every bucket nor see one.
|
||||
let no_permission = s3_client(&endpoint, "gatewaystacknopolicy", USER_SECRET_KEY)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a caller without permission lists nothing");
|
||||
assert_eq!(refusal(&no_permission), (Some("AccessDenied".to_owned()), Some(403)));
|
||||
|
||||
// The root caller is still served by the same server.
|
||||
let listing = s3_client(&endpoint, server.access_key(), server.secret_key())
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect("signed ListBuckets");
|
||||
assert_eq!(listing.buckets().len(), 1);
|
||||
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
/// The S3 request counter belongs to RustFS's layer in front of both stacks, and the operation
|
||||
/// label is set by the code a handler calls. That only works if the gateway polls its handler
|
||||
/// inside that layer's scope, on the request's own task, which is what this observes.
|
||||
#[test]
|
||||
fn the_gateway_stack_counts_each_request_once_under_the_label_the_legacy_stack_uses() {
|
||||
with_stack(Some("gateway"), |server| async move {
|
||||
let endpoint = server.endpoint();
|
||||
|
||||
let before = s3_request_counts();
|
||||
s3_client(&endpoint, server.access_key(), server.secret_key())
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect("signed ListBuckets");
|
||||
assert_eq!(
|
||||
counted_since(&before),
|
||||
BTreeMap::from([(("GET", S3Operation::ListBuckets.as_str(), "2xx"), 1)]),
|
||||
"one served request is counted once, as ListBuckets"
|
||||
);
|
||||
|
||||
let before = s3_request_counts();
|
||||
let refused = s3_client(&endpoint, ROOT_ACCESS_KEY, "not-the-root-secret")
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a wrong secret signs nothing");
|
||||
assert_eq!(refusal(&refused), (Some("SignatureDoesNotMatch".to_owned()), Some(403)));
|
||||
assert_eq!(
|
||||
counted_since(&before),
|
||||
BTreeMap::from([(("GET", "unknown", "4xx"), 1)]),
|
||||
"a request refused by authentication is counted once, under no operation"
|
||||
);
|
||||
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
/// The pinned gateway admits a request that carries only `X-Auth-Token` as anonymous, while the
|
||||
/// RustFS code behind it reads the Keystone identity. Until the gateway authenticates the token
|
||||
/// (rustfs/gateway#1586) the two settings together refuse to start, and nothing is served.
|
||||
#[test]
|
||||
fn the_gateway_stack_refuses_to_start_with_keystone_enabled() {
|
||||
let environment = [
|
||||
(ENV_S3_STACK, Some("gateway")),
|
||||
(ENV_KEYSTONE_ENABLE, Some("true")),
|
||||
(ENV_KEYSTONE_AUTH_URL, Some(UNREACHABLE_KEYSTONE)),
|
||||
];
|
||||
temp_env::with_vars(environment, || {
|
||||
common::run_embedded_test(|| async {
|
||||
let Some(started) = start_server().await else { return };
|
||||
let refusal = match started {
|
||||
Ok(server) => {
|
||||
server.shutdown().await;
|
||||
panic!("the gateway stack started with Keystone enabled");
|
||||
}
|
||||
Err(refusal) => refusal,
|
||||
};
|
||||
for named in ["RUSTFS_S3_STACK=gateway", "RUSTFS_KEYSTONE_ENABLE", "rustfs/gateway#1586"] {
|
||||
assert!(refusal.contains(named), "{named} is missing from: {refusal}");
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_stack_starts_with_keystone_enabled() {
|
||||
let environment = vec![
|
||||
(ENV_S3_STACK, None),
|
||||
(ENV_KEYSTONE_ENABLE, Some("true")),
|
||||
(ENV_KEYSTONE_AUTH_URL, Some(UNREACHABLE_KEYSTONE)),
|
||||
];
|
||||
with_environment(environment, |server| async move {
|
||||
let listing = s3_client(&server.endpoint(), server.access_key(), server.secret_key())
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect("signed ListBuckets on the legacy stack with Keystone enabled");
|
||||
assert!(listing.buckets().is_empty());
|
||||
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_stack_still_serves_every_operation() {
|
||||
with_stack(None, |server| async move {
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket("legacy-stack-alpha")
|
||||
.send()
|
||||
.await
|
||||
.expect("CreateBucket on the legacy stack");
|
||||
client
|
||||
.put_object()
|
||||
.bucket("legacy-stack-alpha")
|
||||
.key("object")
|
||||
.body(ByteStream::from_static(b"payload"))
|
||||
.send()
|
||||
.await
|
||||
.expect("PutObject on the legacy stack");
|
||||
let before = s3_request_counts();
|
||||
let listing = client.list_buckets().send().await.expect("ListBuckets on the legacy stack");
|
||||
assert_eq!(
|
||||
counted_since(&before),
|
||||
BTreeMap::from([(("GET", S3Operation::ListBuckets.as_str(), "2xx"), 1)]),
|
||||
"the label the gateway stack is held to is the one this stack counts under"
|
||||
);
|
||||
let names: Vec<&str> = listing.buckets().iter().filter_map(|bucket| bucket.name()).collect();
|
||||
assert_eq!(names, ["legacy-stack-alpha"]);
|
||||
assert!(listing.owner().and_then(|owner| owner.id()).is_some(), "the listing names its owner");
|
||||
|
||||
let missing = client
|
||||
.get_object()
|
||||
.bucket("legacy-stack-alpha")
|
||||
.key("missing")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a missing key");
|
||||
assert_eq!(refusal(&missing), (Some("NoSuchKey".to_owned()), Some(404)));
|
||||
|
||||
create_user_without_a_policy("legacystacknopolicy").await;
|
||||
let no_permission = s3_client(&server.endpoint(), "legacystacknopolicy", USER_SECRET_KEY)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a caller without permission lists nothing on either stack");
|
||||
assert_eq!(refusal(&no_permission), (Some("AccessDenied".to_owned()), Some(403)));
|
||||
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
@@ -38,6 +38,7 @@ their issue closes.
|
||||
| `check_no_tokio_io_uring.sh` | ci-gate | Keeps tokio's io-uring backend disabled | ci.yml Quick Checks |
|
||||
| `check_s3s_footprint.sh` | ci-gate | Exact, lower-only baseline on the direct s3s surface ahead of the gateway migration (reads `.config/s3s-footprint-baseline.txt`; `s3_error_lines` leaves out lines whose file imports the macro from `rustfs-s3-types`, rule in the script header); `--mode allowlist` fails on any Rust file referencing s3s outside `.config/s3s-edge-allowlist.txt`, `--dry-run` lists them | ci.yml Quick Checks; `make pre-commit`; `make script-tests` (`--self-test`) |
|
||||
| `check_test_count_ratchet.sh` | ci-gate | Per-crate test-count floor and `#[ignore]` ceiling against `.config/test-count-baseline.txt`; no crate may fall below its tests row or rise above its ignored row (extra tests and new crates pass with a note) | ci.yml Quick Checks; `make pre-commit` |
|
||||
| `guard_source_set.sh` | ci-gate (library) | Sourced helper that resolves a text guard's input as a required file plus the directory a mechanical split moves its code into (rustfs/backlog#2765); a missing required file fails | Sourced by `check_architecture_migration_rules.sh` and `check_logging_guardrails.sh`; contract tests `test_guard_source_set.sh` (`make script-tests`) |
|
||||
| `check_unsafe_code_allowances.sh` | ci-gate | Unsafe-code allowance ledger guard | ci.yml Quick Checks |
|
||||
| `layer-dependency-baseline.txt` | ci-gate (data) | Committed baseline consumed by `check_layer_dependencies.sh` | [Architecture guard troubleshooting](../docs/operations/architecture-guard-troubleshooting.md) |
|
||||
| `static.sh` | ci-gate | Static-build helper executed inside image builds | `Dockerfile.source`, `Dockerfile.decommission-local` |
|
||||
|
||||
@@ -118,6 +118,28 @@ require_source_contains "docs/architecture/ecstore-api-facade-inventory.md" "Do
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
# Rules about one of the large files below read that file plus the sibling
|
||||
# directory a mechanical split moves its code into (rustfs/backlog#2765), so a
|
||||
# split neither breaks nor silently blinds them. The *_SPLIT_SOURCES arrays hold
|
||||
# only the directory part, for rules that select the file itself by glob.
|
||||
# shellcheck source=scripts/guard_source_set.sh
|
||||
source "${ROOT_DIR}/scripts/guard_source_set.sh"
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/storage/ecfs.rs rustfs/src/storage/legacy_edge ||
|
||||
report_failure "ECFS guard source set is unreadable"
|
||||
ECFS_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/app/bucket_usecase.rs rustfs/src/app/bucket_usecase ||
|
||||
report_failure "bucket usecase guard source set is unreadable"
|
||||
BUCKET_USECASE_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
BUCKET_USECASE_SPLIT_SOURCES=("${GUARD_SOURCES[@]:1}")
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/app/multipart_usecase.rs rustfs/src/app/multipart_usecase ||
|
||||
report_failure "multipart usecase guard source set is unreadable"
|
||||
MULTIPART_USECASE_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
MULTIPART_USECASE_SPLIT_SOURCES=("${GUARD_SOURCES[@]:1}")
|
||||
guard_source_set "$ROOT_DIR" rustfs/src/app/lifecycle_transition_api_test.rs rustfs/src/app/lifecycle_transition_api_test ||
|
||||
report_failure "lifecycle transition API test guard source set is unreadable"
|
||||
LIFECYCLE_TRANSITION_TEST_GUARD_SOURCES=("${GUARD_SOURCES[@]}")
|
||||
LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES=("${GUARD_SOURCES[@]:1}")
|
||||
|
||||
PR_TYPES_FILE="${TMP_DIR}/pr_types.txt"
|
||||
PR_TYPE_HITS_FILE="${TMP_DIR}/pr_type_hits.txt"
|
||||
SOURCE_MARKERS_FILE="${TMP_DIR}/source_markers.txt"
|
||||
@@ -2157,7 +2179,7 @@ fi
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::app::context::(?:\{[^}]*\b(?:resolve_object_store_handle|resolve_buffer_config|resolve_internode_metrics|resolve_local_node_name|resolve_action_credentials|resolve_notify_interface|resolve_performance_metrics|resolve_encryption_service|resolve_region|resolve_ready_iam_handle)\b|(?:resolve_object_store_handle|resolve_buffer_config|resolve_internode_metrics|resolve_local_node_name|resolve_action_credentials|resolve_notify_interface|resolve_performance_metrics|resolve_encryption_service|resolve_region|resolve_ready_iam_handle)\b)' \
|
||||
rustfs/src/storage/access.rs \
|
||||
rustfs/src/storage/ecfs.rs \
|
||||
"${ECFS_GUARD_SOURCES[@]}" \
|
||||
rustfs/src/storage/ecfs_extend.rs \
|
||||
rustfs/src/storage/helper.rs \
|
||||
rustfs/src/storage/concurrency/manager.rs \
|
||||
@@ -2175,7 +2197,7 @@ fi
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::app::(?:bucket_usecase|multipart_usecase|object_usecase|object\b)|Default(?:Bucket|Multipart|Object)Usecase::from_global\(\)' \
|
||||
rustfs/src/storage/ecfs.rs \
|
||||
"${ECFS_GUARD_SOURCES[@]}" \
|
||||
--glob '*.rs' || true
|
||||
) >"$RUSTFS_STORAGE_ECFS_USECASE_BYPASS_HITS_FILE"
|
||||
|
||||
@@ -2228,7 +2250,10 @@ fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'use crate::storage::\*;' rustfs/src/app --glob '*_usecase.rs' --glob 'object/*.rs' || true
|
||||
# A glob containing a slash is anchored at the search root, so object files
|
||||
# need the leading **/ to match under rustfs/src/app.
|
||||
rg -n --with-filename 'use crate::storage::\*;' rustfs/src/app --glob '*_usecase.rs' --glob '**/object/*.rs' \
|
||||
"${BUCKET_USECASE_SPLIT_SOURCES[@]}" "${MULTIPART_USECASE_SPLIT_SOURCES[@]}" || true
|
||||
) >"$RUSTFS_APP_USECASE_STORAGE_WILDCARD_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_APP_USECASE_STORAGE_WILDCARD_HITS_FILE" ]]; then
|
||||
@@ -2246,7 +2271,8 @@ fi
|
||||
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::storage::s3_api::|use crate::storage::s3_api|super::s3_api::|use super::s3_api' rustfs/src/app --glob '*_usecase.rs' --glob 'object/*.rs' || true
|
||||
rg -n --with-filename 'crate::storage::s3_api::|use crate::storage::s3_api|super::s3_api::|use super::s3_api' rustfs/src/app --glob '*_usecase.rs' --glob '**/object/*.rs' \
|
||||
"${BUCKET_USECASE_SPLIT_SOURCES[@]}" "${MULTIPART_USECASE_SPLIT_SOURCES[@]}" || true
|
||||
) >"$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE"
|
||||
|
||||
if [[ -s "$RUSTFS_APP_USECASE_S3_API_BYPASS_HITS_FILE" ]]; then
|
||||
@@ -2258,13 +2284,14 @@ fi
|
||||
{
|
||||
rg -n --with-filename \
|
||||
'(use crate::storage::(access|helper|options|request_context|sse|timeout_wrapper|head_prefix|concurrency|ecfs)|crate::storage::sse::EncryptionKeyKind|use crate::storage::\{|use crate::storage::[A-Z])' \
|
||||
rustfs/src/app/select_object.rs rustfs/src/app/*_usecase.rs rustfs/src/app/object || true
|
||||
rustfs/src/app/select_object.rs rustfs/src/app/*_usecase.rs rustfs/src/app/object \
|
||||
"${BUCKET_USECASE_SPLIT_SOURCES[@]}" "${MULTIPART_USECASE_SPLIT_SOURCES[@]}" || true
|
||||
rg -n --with-filename \
|
||||
'use super::(?:\{[^}]*\b(?:DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|StorageError|DiskError|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b|(?:object_api_utils::to_s3s_etag|storageclass|StorageError|DiskError|DynReader|HashReader|WriteEncryption|WritePlan|DecryptReader|EncryptReader|HardLimitReader|boxed_reader|wrap_reader|compression_metadata_value|is_disk_compressible|MIN_DISK_COMPRESSIBLE_SIZE|get_lock_acquire_timeout|is_valid_storage_class|is_all_buckets_not_found|is_err_bucket_not_found|is_err_object_not_found|is_err_version_not_found)\b)' \
|
||||
rustfs/src/app/bucket_usecase.rs rustfs/src/app/object_usecase.rs rustfs/src/app/object rustfs/src/app/multipart_usecase.rs rustfs/src/app/lifecycle_transition_api_test.rs || true
|
||||
"${BUCKET_USECASE_GUARD_SOURCES[@]}" rustfs/src/app/object_usecase.rs rustfs/src/app/object "${MULTIPART_USECASE_GUARD_SOURCES[@]}" "${LIFECYCLE_TRANSITION_TEST_GUARD_SOURCES[@]}" || true
|
||||
rg -n --with-filename \
|
||||
'use super::(?:\{[^}]*\b(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b|(?:AppObjectLockConfigExt|AppReplicationConfigExt|AppVersioningConfigExt|predict_lifecycle_expiration|validate_restore_request|bucket_target_sys|lifecycle|metadata|metadata_sys|object_lock|policy_sys|quota|replication|tagging|target|utils|versioning_sys|transition_api|ObjectInfo|ObjectOptions)\b)|super::(?:lifecycle|metadata_sys|object_lock|quota|replication|tagging|target|utils|versioning_sys|transition_api)::|super::super::(?:metadata_sys|lifecycle|target)::' \
|
||||
rustfs/src/app/bucket_usecase.rs rustfs/src/app/object_usecase.rs rustfs/src/app/object rustfs/src/app/multipart_usecase.rs rustfs/src/app/lifecycle_transition_api_test.rs rustfs/src/app/capacity_dirty_scope_test.rs rustfs/src/app/context.rs rustfs/src/app/context/handles.rs rustfs/src/app/context/interfaces.rs rustfs/src/app/context/runtime_sources.rs || true
|
||||
"${BUCKET_USECASE_GUARD_SOURCES[@]}" rustfs/src/app/object_usecase.rs rustfs/src/app/object "${MULTIPART_USECASE_GUARD_SOURCES[@]}" "${LIFECYCLE_TRANSITION_TEST_GUARD_SOURCES[@]}" rustfs/src/app/capacity_dirty_scope_test.rs rustfs/src/app/context.rs rustfs/src/app/context/handles.rs rustfs/src/app/context/interfaces.rs rustfs/src/app/context/runtime_sources.rs || true
|
||||
}
|
||||
) >"$RUSTFS_APP_USECASE_STORAGE_API_BYPASS_HITS_FILE"
|
||||
|
||||
@@ -2277,6 +2304,7 @@ fi
|
||||
rg -n --with-filename \
|
||||
'(use crate::storage::(access|request_context|ecfs)|use crate::storage::\{[^}]*Storage(ObjectInfo|ObjectOptions|PutObjReader)|crate::storage::StorageObjectOptions)' \
|
||||
rustfs/src/app/*_test.rs \
|
||||
"${LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES[@]}" \
|
||||
rustfs/src/admin/router.rs \
|
||||
rustfs/src/admin/console.rs \
|
||||
rustfs/src/admin/handlers/heal.rs \
|
||||
@@ -4561,7 +4589,7 @@ fi
|
||||
cd "$ROOT_DIR"
|
||||
rg -n --with-filename 'crate::storage::core_storage_compat' \
|
||||
rustfs/src/storage/mod.rs \
|
||||
rustfs/src/storage/ecfs.rs \
|
||||
"${ECFS_GUARD_SOURCES[@]}" \
|
||||
rustfs/src/storage/ecfs_extend.rs \
|
||||
rustfs/src/storage/access.rs \
|
||||
rustfs/src/storage/head_prefix.rs \
|
||||
@@ -4577,7 +4605,8 @@ fi
|
||||
cd "$ROOT_DIR"
|
||||
{
|
||||
rg -n --with-filename 'crate::app::usecase_storage_compat' \
|
||||
rustfs/src/app/*.rs || true
|
||||
rustfs/src/app/*.rs "${BUCKET_USECASE_SPLIT_SOURCES[@]}" "${MULTIPART_USECASE_SPLIT_SOURCES[@]}" \
|
||||
"${LIFECYCLE_TRANSITION_TEST_SPLIT_SOURCES[@]}" || true
|
||||
rg -n --with-filename 'crate::admin::router_storage_compat' \
|
||||
rustfs/src/admin/router.rs || true
|
||||
rg -n --with-filename 'crate::storage::core_storage_compat' \
|
||||
|
||||
@@ -1,226 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Keeps the `gateway-stack` cargo feature of the rustfs crate out of every release build while the
|
||||
# gateway S3 stack is under construction (rustfs/backlog#2734 T2.5).
|
||||
#
|
||||
# Usage: scripts/check_gateway_stack_release_boundary.sh [--root DIR]
|
||||
# scripts/check_gateway_stack_release_boundary.sh --self-test
|
||||
#
|
||||
# Three facts are checked, none of which needs Cargo:
|
||||
# 1. rustfs/src/server/gateway_stack.rs refuses to compile without debug assertions, so a
|
||||
# release-profile build that enables the feature fails in the compiler;
|
||||
# 2. neither the `default` nor the `full` feature list of rustfs/Cargo.toml enables it, directly
|
||||
# or through a feature that enables it, and the feature itself still exists (a guard whose
|
||||
# subject is gone must fail, not skip);
|
||||
# 3. no file that produces a release artifact names the feature or one that enables it.
|
||||
# A missing input is an error: every file below exists in this repository.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
FEATURE="gateway-stack"
|
||||
GUARD_SOURCE="rustfs/src/server/gateway_stack.rs"
|
||||
GUARD_CFG='#[cfg(not(debug_assertions))]'
|
||||
GUARD_ERROR='compile_error!("gateway-stack is restricted to debug builds");'
|
||||
MANIFEST="rustfs/Cargo.toml"
|
||||
# Everything that builds or packages a release binary.
|
||||
RELEASE_FILES=(
|
||||
".github/workflows/build.yml"
|
||||
".github/workflows/docker.yml"
|
||||
"build-rustfs.sh"
|
||||
"docker-buildx.sh"
|
||||
"Dockerfile"
|
||||
"Dockerfile.glibc"
|
||||
"Dockerfile.source"
|
||||
"rustfs.spec"
|
||||
"Justfile"
|
||||
".config/make/build.mak"
|
||||
".config/make/build-docker.mak"
|
||||
)
|
||||
|
||||
fail() {
|
||||
echo "gateway-stack release boundary: $*" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# Prints the value of one `name = [...]` feature line of the manifest.
|
||||
feature_list() {
|
||||
local manifest="$1" name="$2"
|
||||
grep -E "^${name}[[:space:]]*=" "$manifest" || true
|
||||
}
|
||||
|
||||
# Prints the feature and every feature that enables it, directly or through another, one per line.
|
||||
enabling_features() {
|
||||
local manifest="$1" known="$FEATURE" grown name
|
||||
while true; do
|
||||
grown="$known"
|
||||
while IFS= read -r name; do
|
||||
grep -qxF -- "$name" <<<"$grown" || grown+=$'\n'"$name"
|
||||
done < <(
|
||||
while IFS= read -r name; do
|
||||
grep -E "^[A-Za-z0-9_-]+[[:space:]]*=.*\"${name}\"" "$manifest" | sed -E 's/[[:space:]]*=.*//' || true
|
||||
done <<<"$known"
|
||||
)
|
||||
[[ "$grown" == "$known" ]] && break
|
||||
known="$grown"
|
||||
done
|
||||
printf '%s\n' "$known"
|
||||
}
|
||||
|
||||
check_root() {
|
||||
local root="$1" status=0 file line name enabler
|
||||
local -a enablers=("$FEATURE")
|
||||
|
||||
file="${root}/${GUARD_SOURCE}"
|
||||
if [[ ! -f "$file" ]]; then
|
||||
fail "${GUARD_SOURCE} is missing" || status=1
|
||||
else
|
||||
# The cfg must sit on the line directly above the compile_error.
|
||||
line="$(grep -nF "$GUARD_ERROR" "$file" | head -n 1 | cut -d: -f1)"
|
||||
if [[ -z "$line" ]]; then
|
||||
fail "${GUARD_SOURCE} no longer refuses a build without debug assertions" || status=1
|
||||
elif [[ "$(sed -n "$((line - 1))p" "$file" | tr -d '[:space:]')" != "$(tr -d '[:space:]' <<<"$GUARD_CFG")" ]]; then
|
||||
fail "the compile_error in ${GUARD_SOURCE} is not gated on cfg(not(debug_assertions))" || status=1
|
||||
fi
|
||||
fi
|
||||
|
||||
file="${root}/${MANIFEST}"
|
||||
if [[ ! -f "$file" ]]; then
|
||||
fail "${MANIFEST} is missing" || status=1
|
||||
else
|
||||
if [[ -z "$(feature_list "$file" "$FEATURE")" ]]; then
|
||||
fail "${MANIFEST} no longer declares the ${FEATURE} feature this guard protects" || status=1
|
||||
fi
|
||||
mapfile -t enablers < <(enabling_features "$file")
|
||||
for name in default full; do
|
||||
line="$(feature_list "$file" "$name")"
|
||||
if [[ -z "$line" ]]; then
|
||||
fail "${MANIFEST} has no ${name} feature list" || status=1
|
||||
fi
|
||||
for enabler in "${enablers[@]}"; do
|
||||
if [[ "$enabler" == "$name" ]]; then
|
||||
fail "the ${name} feature list of ${MANIFEST} enables ${FEATURE}" || status=1
|
||||
fi
|
||||
done
|
||||
done
|
||||
fi
|
||||
|
||||
for file in "${RELEASE_FILES[@]}"; do
|
||||
if [[ ! -f "${root}/${file}" ]]; then
|
||||
fail "release file ${file} is missing; update RELEASE_FILES" || status=1
|
||||
continue
|
||||
fi
|
||||
for enabler in "${enablers[@]}"; do
|
||||
if grep -qF -- "$enabler" "${root}/${file}"; then
|
||||
fail "${file} names ${enabler}, which enables ${FEATURE}; release artifacts must not" || status=1
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
return "$status"
|
||||
}
|
||||
|
||||
self_test() {
|
||||
local scratch fixture failures=0
|
||||
scratch="$(mktemp -d)"
|
||||
|
||||
make_fixture() {
|
||||
fixture="${scratch}/$1"
|
||||
mkdir -p "${fixture}/rustfs/src/server" "${fixture}/.github/workflows" "${fixture}/.config/make"
|
||||
printf '%s\n%s\n' "$GUARD_CFG" "$GUARD_ERROR" >"${fixture}/${GUARD_SOURCE}"
|
||||
printf '%s\n' '[features]' 'default = ["ftps"]' 'full = ["ftps", "swift"]' 'gateway-stack = ["dep:rustfs-gateway"]' \
|
||||
>"${fixture}/${MANIFEST}"
|
||||
for file in "${RELEASE_FILES[@]}"; do
|
||||
printf 'cargo build --release -p rustfs\n' >"${fixture}/${file}"
|
||||
done
|
||||
}
|
||||
|
||||
expect() {
|
||||
local wanted="$1" label="$2"
|
||||
if check_root "$fixture" >/dev/null 2>&1; then
|
||||
[[ "$wanted" == "pass" ]] || { echo "self-test: ${label} was accepted" >&2; failures=$((failures + 1)); }
|
||||
else
|
||||
[[ "$wanted" == "fail" ]] || { echo "self-test: ${label} was refused" >&2; failures=$((failures + 1)); }
|
||||
fi
|
||||
}
|
||||
|
||||
make_fixture valid
|
||||
expect pass "a repository that keeps the boundary"
|
||||
|
||||
make_fixture no-compile-error
|
||||
printf '// nothing here\n' >"${fixture}/${GUARD_SOURCE}"
|
||||
expect fail "a gateway stack without the compile-time refusal"
|
||||
|
||||
make_fixture ungated-compile-error
|
||||
printf '%s\n%s\n' '#[cfg(test)]' "$GUARD_ERROR" >"${fixture}/${GUARD_SOURCE}"
|
||||
expect fail "a compile_error gated on something else"
|
||||
|
||||
make_fixture missing-source
|
||||
rm "${fixture}/${GUARD_SOURCE}"
|
||||
expect fail "a missing guard source"
|
||||
|
||||
make_fixture default-enables
|
||||
sed -i.bak 's/^default = .*/default = ["ftps", "gateway-stack"]/' "${fixture}/${MANIFEST}"
|
||||
expect fail "a default feature list that enables the stack"
|
||||
|
||||
make_fixture full-enables
|
||||
sed -i.bak 's/^full = .*/full = ["ftps", "gateway-stack"]/' "${fixture}/${MANIFEST}"
|
||||
expect fail "a full feature list that enables the stack"
|
||||
|
||||
make_fixture feature-gone
|
||||
sed -i.bak '/^gateway-stack = /d' "${fixture}/${MANIFEST}"
|
||||
expect fail "a manifest without the feature"
|
||||
|
||||
make_fixture full-enables-through-another
|
||||
printf '%s\n' 'hooks = ["gateway-stack"]' >>"${fixture}/${MANIFEST}"
|
||||
sed -i.bak 's/^full = .*/full = ["ftps", "hooks"]/' "${fixture}/${MANIFEST}"
|
||||
expect fail "a full feature list that enables the stack through another feature"
|
||||
|
||||
make_fixture debug-only-enabler
|
||||
printf '%s\n' 'hooks = ["gateway-stack"]' >>"${fixture}/${MANIFEST}"
|
||||
expect pass "a feature outside default and full that enables the stack"
|
||||
|
||||
make_fixture release-file-names-enabler
|
||||
printf '%s\n' 'hooks = ["gateway-stack"]' >>"${fixture}/${MANIFEST}"
|
||||
printf 'cargo build --release -p rustfs --features hooks\n' >"${fixture}/build-rustfs.sh"
|
||||
expect fail "a release file that enables the stack through another feature"
|
||||
|
||||
make_fixture release-workflow
|
||||
printf 'cargo build --release -p rustfs --features gateway-stack\n' >"${fixture}/.github/workflows/build.yml"
|
||||
expect fail "a release workflow that enables the stack"
|
||||
|
||||
make_fixture missing-release-file
|
||||
rm "${fixture}/build-rustfs.sh"
|
||||
expect fail "a missing release file"
|
||||
|
||||
rm -rf "$scratch"
|
||||
if ((failures > 0)); then
|
||||
echo "gateway-stack release boundary self-test: ${failures} failure(s)" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "gateway-stack release boundary self-test passed"
|
||||
}
|
||||
|
||||
main() {
|
||||
local root
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
case "${1:-}" in
|
||||
--self-test)
|
||||
[[ $# -eq 1 ]] || { echo "usage: $0 [--root DIR] | --self-test" >&2; exit 2; }
|
||||
self_test
|
||||
;;
|
||||
--root)
|
||||
[[ $# -eq 2 ]] || { echo "usage: $0 [--root DIR] | --self-test" >&2; exit 2; }
|
||||
check_root "$2"
|
||||
echo "gateway-stack release boundary holds"
|
||||
;;
|
||||
"")
|
||||
check_root "$root"
|
||||
echo "gateway-stack release boundary holds"
|
||||
;;
|
||||
*)
|
||||
echo "usage: $0 [--root DIR] | --self-test" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -22,7 +22,8 @@ classify_source_layer() {
|
||||
printf 'composition'
|
||||
elif [[ "$file" == rustfs/src/app/* ]]; then
|
||||
printf 'app'
|
||||
elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]] ||
|
||||
elif [[ "$file" == rustfs/src/admin/* ]] || [[ "$file" == rustfs/src/storage/ecfs.rs ]] ||
|
||||
[[ "$file" == rustfs/src/storage/legacy_edge/* ]] || [[ "$file" == rustfs/src/storage/s3_api/* ]] ||
|
||||
[[ "$file" == rustfs/src/storage/s3_compat/* ]]; then
|
||||
printf 'interface'
|
||||
elif [[ "$file" == rustfs/src/* ]]; then
|
||||
@@ -55,6 +56,7 @@ classify_target_layer() {
|
||||
storage)
|
||||
storage_path="${path#storage::}"
|
||||
if [[ "$storage_path" == "ecfs" ]] || [[ "$storage_path" == ecfs::* ]] ||
|
||||
[[ "$storage_path" == "legacy_edge" ]] || [[ "$storage_path" == legacy_edge::* ]] ||
|
||||
[[ "$storage_path" == "s3_api" ]] || [[ "$storage_path" == s3_api::* ]] ||
|
||||
[[ "$storage_path" == "s3_compat" ]] || [[ "$storage_path" == s3_compat::* ]]; then
|
||||
printf 'interface'
|
||||
@@ -137,6 +139,60 @@ run_layer_model_self_tests() {
|
||||
assert_dependency_direction 'allowed' "$s3_compat_source" "$app_target"
|
||||
assert_dependency_direction 'reverse' "$app_source" "$s3_compat_target"
|
||||
assert_dependency_direction 'reverse' "$storage_source" "$s3_compat_target"
|
||||
|
||||
# A mechanical split moves ecfs.rs code into storage/legacy_edge/ and the
|
||||
# usecase files' code into same-named directories; each keeps its layer.
|
||||
assert_source_layer 'interface' rustfs/src/storage/ecfs.rs
|
||||
assert_source_layer 'interface' rustfs/src/storage/legacy_edge/bucket_core.rs
|
||||
assert_source_layer 'interface' rustfs/src/storage/legacy_edge/tests/object_meta.rs
|
||||
assert_source_layer 'infra' rustfs/src/storage/legacy_edge.rs
|
||||
assert_source_layer 'infra' rustfs/src/storage/legacy_edge_helper.rs
|
||||
assert_source_layer 'app' rustfs/src/app/bucket_usecase/cors.rs
|
||||
assert_source_layer 'app' rustfs/src/app/multipart_usecase/complete.rs
|
||||
assert_target_layer 'interface' storage::legacy_edge
|
||||
assert_target_layer 'interface' storage::legacy_edge::object_meta
|
||||
assert_target_layer 'infra' storage::legacy_edge_helper
|
||||
assert_dependency_direction 'allowed' "$(classify_source_layer rustfs/src/storage/legacy_edge/bucket_core.rs)" "$app_target"
|
||||
assert_dependency_direction 'reverse' "$app_source" "$(classify_target_layer storage::legacy_edge::bucket_core)"
|
||||
assert_dependency_direction 'reverse' "$storage_source" "$(classify_target_layer storage::legacy_edge)"
|
||||
assert_dedicated_test_file 'yes' rustfs/src/app/lifecycle_transition_api_test.rs
|
||||
assert_dedicated_test_file 'yes' rustfs/src/app/lifecycle_transition_api_test/put.rs
|
||||
assert_dedicated_test_file 'no' rustfs/src/app/lifecycle_transition_api_test_support.rs
|
||||
assert_dedicated_test_file 'no' rustfs/src/app/lifecycle_transition_api.rs
|
||||
assert_dedicated_test_file 'no' rustfs/src/app/bucket_usecase/cors.rs
|
||||
assert_dedicated_test_file 'no' rustfs/src/storage/legacy_edge/bucket_core.rs
|
||||
}
|
||||
|
||||
assert_source_layer() {
|
||||
local expected="$1" file="$2" actual
|
||||
|
||||
actual="$(classify_source_layer "$file")"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
printf 'Layer dependency guard self-test failed: %s classified as %s (expected %s)\n' "$file" "$actual" "$expected" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
assert_target_layer() {
|
||||
local expected="$1" path="$2" actual
|
||||
|
||||
actual="$(classify_target_layer "$path")"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
printf 'Layer dependency guard self-test failed: crate::%s classified as %s (expected %s)\n' "$path" "$actual" "$expected" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
assert_dedicated_test_file() {
|
||||
local expected="$1" file="$2" actual='no'
|
||||
|
||||
if is_dedicated_test_file "$file"; then
|
||||
actual='yes'
|
||||
fi
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
printf 'Layer dependency guard self-test failed: %s dedicated test file = %s (expected %s)\n' "$file" "$actual" "$expected" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
normalize_import_group_item() {
|
||||
@@ -251,11 +307,20 @@ normalize_import_path() {
|
||||
printf '%s\n' "$path"
|
||||
}
|
||||
|
||||
# The guard is file-scoped: dedicated test modules are excluded, while inline
|
||||
# #[cfg(test)] imports remain subject to the source file's layer. The
|
||||
# lifecycle_transition_api_test/ directory is where a mechanical split of
|
||||
# lifecycle_transition_api_test.rs moves its tests (rustfs/backlog#2765).
|
||||
is_dedicated_test_file() {
|
||||
local file="$1"
|
||||
|
||||
[[ "$file" == *_test.rs ]] || [[ "$file" == */tests/* ]] ||
|
||||
[[ "$file" == rustfs/src/app/lifecycle_transition_api_test/* ]]
|
||||
}
|
||||
|
||||
emit_crate_use_statements() {
|
||||
(cd "$ROOT_DIR" && rg --files -g '*.rs' rustfs/src | while IFS= read -r file; do
|
||||
# The guard is file-scoped: dedicated test modules are excluded, while
|
||||
# inline #[cfg(test)] imports remain subject to the source file's layer.
|
||||
if [[ "$file" == *_test.rs ]] || [[ "$file" == */tests/* ]]; then
|
||||
if is_dedicated_test_file "$file"; then
|
||||
continue
|
||||
fi
|
||||
perl -0777 -ne '
|
||||
|
||||
@@ -1094,11 +1094,29 @@ trace_hot_spans=(
|
||||
"rustfs/src/app/object:execute_get_object"
|
||||
)
|
||||
|
||||
# A hot span in a file that is due for a mechanical split (rustfs/backlog#2765)
|
||||
# is also looked up in the directory the split moves its functions into; the
|
||||
# named file itself stays required.
|
||||
# shellcheck source=scripts/guard_source_set.sh
|
||||
source "${repo_root}/scripts/guard_source_set.sh"
|
||||
|
||||
for hot_span in "${trace_hot_spans[@]}"; do
|
||||
file="${hot_span%%:*}"
|
||||
function="${hot_span##*:}"
|
||||
hot_span_sources=("$file")
|
||||
split_dir=""
|
||||
case "$file" in
|
||||
rustfs/src/app/bucket_usecase.rs) split_dir="rustfs/src/app/bucket_usecase" ;;
|
||||
esac
|
||||
if [[ -n "$split_dir" ]]; then
|
||||
if ! guard_source_set "$repo_root" "$file" "$split_dir"; then
|
||||
echo "❌ logging guardrail violation: $file::$function guard source is missing" >&2
|
||||
exit 1
|
||||
fi
|
||||
hot_span_sources=("${GUARD_SOURCES[@]}")
|
||||
fi
|
||||
trace_span_pattern="#\\[(tracing::)?instrument\\([^]]*level = \\\"trace\\\"[^]]*\\)\\]([[:space:]]+#\\[[^]]+\\])*[[:space:]]+(pub(\\([^)]*\\))?[[:space:]]+)?(super[[:space:]]+)?async fn ${function}\\b"
|
||||
if ! rg -U "$trace_span_pattern" "$file" >/dev/null; then
|
||||
if ! rg -U "$trace_span_pattern" "${hot_span_sources[@]}" >/dev/null; then
|
||||
echo "❌ logging guardrail violation: $file::$function must remain TRACE-only" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
+2
-4
@@ -17,7 +17,7 @@ ROOT = Path(__file__).resolve().parent.parent
|
||||
ALWAYS_JOBS = ("classify-changes", "typos", "quick-checks")
|
||||
CODE_JOBS = (
|
||||
"workspace-lint-and-build", "workspace-tests", "core-contract", "connect-feature-boundaries", "fips-provider", "test-and-lint-rio-v2",
|
||||
"test-and-lint-protocols", "test-and-lint-gateway-stack", "build-rustfs-debug-binary", "uring-integration",
|
||||
"test-and-lint-protocols", "build-rustfs-debug-binary", "uring-integration",
|
||||
"e2e-tests", "s3-implemented-tests", "s3-lifecycle-behavior-tests",
|
||||
)
|
||||
# Specialized boundary, hardware and alternate-feature lanes do not gate a
|
||||
@@ -33,7 +33,7 @@ NIGHTLY_JOBS = ("connect-feature-boundaries", "fips-provider", "uring-integratio
|
||||
# against this revision's expected job contract. The workflow refuses a plan
|
||||
# that does not carry the current marker and falls back to the full matrix, so
|
||||
# a base planner predating the contract cannot authorize a narrower selection.
|
||||
PLAN_CONTRACT = "4"
|
||||
PLAN_CONTRACT = "3"
|
||||
# These jobs still exercise the server and its black-box harness when only E2E
|
||||
# Rust sources change. Production code, manifests and shared test configuration
|
||||
# always select the full matrix. No production package depends on e2e_test.
|
||||
@@ -299,8 +299,6 @@ def configurations(jobs: dict[str, str]) -> list[dict]:
|
||||
("protocol-sftp-tests", "test-and-lint-protocols", "test", ["sftp"]),
|
||||
("protocol-http3-lint", "test-and-lint-protocols", "dev", ["http3", "swift"]),
|
||||
("protocol-http3-tests", "test-and-lint-protocols", "test", ["http3", "swift"]),
|
||||
("gateway-stack-lint", "test-and-lint-gateway-stack", "dev", ["gateway-stack"]),
|
||||
("gateway-stack-tests", "test-and-lint-gateway-stack", "test", ["gateway-stack"]),
|
||||
("server-rio-v2", "build-rustfs-debug-binary-rio-v2", "dev", ["e2e-test-hooks", "rio-v2"]),
|
||||
]
|
||||
return [{"id": identity, "job": job, "target": "x86_64-unknown-linux-gnu", "profile": profile,
|
||||
|
||||
Executable
+994
@@ -0,0 +1,994 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Split rustfs/src/app/multipart_usecase.rs into one file per operation.
|
||||
|
||||
Preparatory refactor P3 of the s3s to rustfs-gateway migration. A pure code
|
||||
move: every top-level item, every method of `impl DefaultMultipartUsecase` and
|
||||
every test is assigned to a target file by the explicit tables below, and is
|
||||
moved verbatim with its attributes and doc comments. Nothing is renamed and no
|
||||
body changes, with these exceptions, each mechanical and listed here so a
|
||||
reviewer knows where the output differs from the input:
|
||||
|
||||
* the three test fixtures (`s3_op_total`, `build_request`, `make_usecase`)
|
||||
gain `pub(super)` because they move to `test_support.rs`;
|
||||
* the test module's `use` items become `pub(super) use` re-exports in
|
||||
`test_support.rs` (names the parent already imports are not repeated), so
|
||||
every per-operation test module reaches them with one glob;
|
||||
* code-position `s3s::` paths in the new files become
|
||||
`crate::app::storage_api::s3::`, the sanctioned s3s funnel, so the number
|
||||
of files naming s3s (scripts/check_s3s_footprint.sh) does not grow. The
|
||||
funnel gains two re-exports, `dto` and `xml`, in rustfs/src/app/storage_api.rs;
|
||||
* the nextest filter that names the one test it pins follows the test to its
|
||||
new module path.
|
||||
|
||||
Layout produced (parent keeps the struct, its constructors and accessors, the
|
||||
shared helpers and the test fixtures):
|
||||
|
||||
multipart_usecase.rs shared state and helpers
|
||||
multipart_usecase/test_support.rs fixtures and test-only imports
|
||||
multipart_usecase/{abort,create,list,upload_part,complete,upload_part_copy}.rs
|
||||
one operation: its helpers, its
|
||||
`impl DefaultMultipartUsecase` block,
|
||||
its tests
|
||||
multipart_usecase/upload_part/tests/body_read_tests.rs
|
||||
multipart_usecase/upload_part_copy/tests/copy_admission_tests.rs
|
||||
the two former test files, unchanged
|
||||
|
||||
Usage:
|
||||
scripts/codemods/split_multipart_usecase.py [--no-fmt] [--report]
|
||||
scripts/codemods/split_multipart_usecase.py --self-test
|
||||
|
||||
The input is the unsplit tree. A tree that is already split is refused, as is
|
||||
any top-level item, method or test without an assignment, any table entry that
|
||||
matches nothing, and any private item that a file other than its own refers
|
||||
to. Re-running therefore means restoring the pre-split files first:
|
||||
|
||||
git checkout <pre-split-commit> -- rustfs/src/app/multipart_usecase.rs \\
|
||||
rustfs/src/app/multipart_usecase rustfs/src/app/storage_api.rs .config/nextest.toml
|
||||
git clean -fd rustfs/src/app/multipart_usecase
|
||||
|
||||
and then running this script. rustfmt formats the written files unless
|
||||
--no-fmt is given. --report prints the item to file assignment and the
|
||||
cross-file reference check and writes nothing.
|
||||
|
||||
Exit status: 0 on success, 1 when the input cannot be split as asked (the
|
||||
message says why), 2 for a usage error.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
APP = "rustfs/src/app"
|
||||
SOURCE = f"{APP}/multipart_usecase.rs"
|
||||
OUT_DIR = f"{APP}/multipart_usecase"
|
||||
STORAGE_API = f"{APP}/storage_api.rs"
|
||||
NEXTEST = ".config/nextest.toml"
|
||||
STRUCT = "DefaultMultipartUsecase"
|
||||
FUNNEL = "crate::app::storage_api::s3::"
|
||||
|
||||
PARENT = "parent"
|
||||
SUPPORT = "test_support"
|
||||
OPS = ("abort", "create", "list", "upload_part", "complete", "upload_part_copy")
|
||||
OP_DOC = {
|
||||
"abort": "AbortMultipartUpload.",
|
||||
"create": "CreateMultipartUpload.",
|
||||
"list": "ListMultipartUploads and ListParts.",
|
||||
"upload_part": "UploadPart.",
|
||||
"complete": "CompleteMultipartUpload.",
|
||||
"upload_part_copy": "UploadPartCopy.",
|
||||
}
|
||||
|
||||
# Test files that already existed, moved unchanged: (old path, operation).
|
||||
OLD_TEST_FILES = {
|
||||
"body_read_tests": (f"{OUT_DIR}/tests/body_read_tests.rs", "upload_part"),
|
||||
"copy_admission_tests": (f"{OUT_DIR}/tests/copy_admission_tests.rs", "upload_part_copy"),
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Assignment tables: item name -> target file. Every name must be listed; the
|
||||
# script fails on a missing or an unused entry.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def assigned(target, names):
|
||||
return {name: target for name in names.split()}
|
||||
|
||||
|
||||
# Top-level functions (everything that is not an import, the struct, its impl
|
||||
# or the test module). A helper stays in the parent when more than one
|
||||
# operation, or a cross-operation test, uses it.
|
||||
FUNCTIONS = {
|
||||
**assigned(
|
||||
PARENT,
|
||||
"""
|
||||
merge_part_encryption_metadata multipart_plaintext_size multipart_part_numbers
|
||||
multipart_max_total_object_size validate_table_catalog_object_mutation
|
||||
internal_object_info_lookup_opts multipart_upload_api_error
|
||||
""",
|
||||
),
|
||||
**assigned(
|
||||
"upload_part_copy",
|
||||
"normalize_copy_source_version_id format_copy_source_version_id validate_copy_source_range_not_exceeds",
|
||||
),
|
||||
**assigned(
|
||||
"complete",
|
||||
"""
|
||||
validate_complete_multipart_parts normalize_complete_multipart_parts complete_part_from_s3
|
||||
has_complete_multipart_object_lock_headers validate_complete_multipart_checksum_type
|
||||
quota_accounting_object_size encode_s3_path extract_request_scheme extract_request_host
|
||||
build_complete_multipart_location
|
||||
""",
|
||||
),
|
||||
**assigned("create", "create_multipart_upload_metadata should_advertise_session_compression"),
|
||||
**assigned(
|
||||
"upload_part",
|
||||
"""
|
||||
resolve_upload_part_size require_upload_part_size upload_part_body_read_timeout
|
||||
passthrough_part_actual_size normalize_presigned_part_checksums
|
||||
""",
|
||||
),
|
||||
}
|
||||
|
||||
# Methods of `impl DefaultMultipartUsecase`.
|
||||
METHODS = {
|
||||
**assigned(
|
||||
PARENT,
|
||||
"without_context from_global with_context with_context_and_concurrency_manager"
|
||||
" bucket_metadata_sys object_store object_data_cache concurrency_manager",
|
||||
),
|
||||
**assigned("abort", "execute_abort_multipart_upload"),
|
||||
**assigned("complete", "execute_complete_multipart_upload execute_complete_multipart_upload_owned"),
|
||||
**assigned("create", "execute_create_multipart_upload"),
|
||||
**assigned("upload_part", "execute_upload_part"),
|
||||
**assigned("list", "execute_list_multipart_uploads execute_list_parts"),
|
||||
**assigned("upload_part_copy", "execute_upload_part_copy"),
|
||||
}
|
||||
|
||||
# Items of `mod tests`. A test that drives more than one operation, and the
|
||||
# tests of the shared helpers, stay in the parent's own `mod tests`.
|
||||
TESTS = {
|
||||
**assigned(SUPPORT, "s3_op_total build_request make_usecase"),
|
||||
**assigned(
|
||||
PARENT,
|
||||
"""
|
||||
internal_object_info_lookup_opts_drops_http_preconditions
|
||||
merge_part_encryption_metadata_keeps_source_metadata_unchanged
|
||||
managed_multipart_roundtrip_preserves_session_nonce_between_parts
|
||||
multipart_upload_rejects_reserved_directory_markers
|
||||
multipart_max_total_object_size_reads_compatible_internal_metadata
|
||||
multipart_upload_api_error_reports_malformed_upload_id_as_no_such_upload
|
||||
multipart_handlers_reject_another_callers_denied_kms_upload
|
||||
""",
|
||||
),
|
||||
**assigned("abort", "execute_abort_multipart_upload_returns_internal_error_when_store_uninitialized"),
|
||||
**assigned(
|
||||
"create",
|
||||
"""
|
||||
session_compression_is_advertised_only_for_non_passthrough_compressible_uploads
|
||||
execute_create_multipart_upload_rejects_invalid_storage_class
|
||||
create_multipart_upload_metadata_persists_dto_storage_class_without_raw_header
|
||||
create_multipart_upload_metadata_keeps_user_and_system_namespaces_separate
|
||||
create_multipart_rejects_ciphertext_replication_before_parts_are_staged
|
||||
""",
|
||||
),
|
||||
**assigned(
|
||||
"list",
|
||||
"""
|
||||
execute_list_multipart_uploads_returns_internal_error_when_store_uninitialized
|
||||
execute_list_multipart_uploads_rejects_invalid_key_marker_before_store_lookup
|
||||
execute_list_multipart_uploads_rejects_invalid_max_uploads_before_store_lookup
|
||||
execute_list_parts_returns_internal_error_when_store_uninitialized
|
||||
execute_list_parts_rejects_negative_part_number_marker_before_store_lookup
|
||||
execute_list_parts_rejects_invalid_max_parts_before_store_lookup
|
||||
""",
|
||||
),
|
||||
**assigned(
|
||||
"upload_part",
|
||||
"""
|
||||
presigned_checksum_request
|
||||
normalize_presigned_part_checksums_validates_body
|
||||
normalize_presigned_part_checksums_accepts_sdk_crc32_default
|
||||
normalize_presigned_part_checksums_rejects_ambiguous_values
|
||||
normalize_presigned_part_checksums_preserves_headers_and_rejects_invalid_values
|
||||
normalize_presigned_part_checksums_requires_verified_presign
|
||||
resolve_upload_part_size_uses_normalized_logical_length
|
||||
upload_part_length_contract_rejects_unknown_and_all_negative_lengths
|
||||
upload_part_timeout_policy_preserves_disabled_and_capped_floor
|
||||
execute_upload_part_rejects_missing_body
|
||||
execute_upload_part_rejects_oversize_declared_part_before_reading_the_body
|
||||
execute_upload_part_rejects_invalid_part_number_before_body_lookup
|
||||
execute_upload_part_rejects_unknown_length_before_admission_or_body_polling
|
||||
execute_upload_part_progress_timeout_releases_admission_and_leaves_no_part
|
||||
execute_upload_part_rejects_when_foreground_write_admission_is_full
|
||||
""",
|
||||
),
|
||||
**assigned(
|
||||
"upload_part_copy",
|
||||
"""
|
||||
execute_upload_part_copy_returns_internal_error_when_store_uninitialized
|
||||
execute_upload_part_copy_rejects_invalid_part_number_before_store_lookup
|
||||
test_validate_copy_source_range_not_exceeds_returns_invalid_range_when_range_exceeds
|
||||
test_validate_copy_source_range_not_exceeds_ok_when_range_valid
|
||||
test_validate_copy_source_range_not_exceeds_ok_for_suffix_range
|
||||
upload_part_copy_normalizes_and_formats_source_version_id
|
||||
""",
|
||||
),
|
||||
**assigned(
|
||||
"complete",
|
||||
"""
|
||||
upload_metadata_with_checksum_type headers_with_checksum_type
|
||||
complete_multipart_checksum_type_matching_the_upload_is_accepted
|
||||
complete_multipart_checksum_type_contradicting_the_upload_is_rejected
|
||||
complete_multipart_checksum_type_without_both_sides_is_accepted
|
||||
quota_accounting_uses_logical_size_when_available
|
||||
quota_accounting_fails_closed_only_when_quota_is_configured
|
||||
test_build_complete_multipart_location_uses_forwarded_proto_and_encodes_key
|
||||
test_build_complete_multipart_location_falls_back_to_uri_authority_and_scheme
|
||||
test_build_complete_multipart_location_returns_path_without_host
|
||||
execute_complete_multipart_upload_rejects_missing_parts_payload
|
||||
execute_complete_multipart_upload_rejects_missing_parts_list
|
||||
compressed_complete_records_logical_quota_usage_and_overwrite_delta
|
||||
authorized_replica_only_multipart_complete_preserves_persisted_anti_cascade_state
|
||||
authorized_replica_only_multipart_complete_preserves_persisted_anti_cascade_state_inner
|
||||
ssec_completion_and_replay_require_key_before_disclosing_plaintext_checksums
|
||||
assert_multipart_etag_listings
|
||||
kms_multipart_completion_and_replay_preserve_etag_in_versioned_listings
|
||||
conditional_multipart_upload_if_none_match_returns_precondition_failed_when_object_exists
|
||||
conditional_multipart_upload_if_none_match_returns_precondition_failed_when_object_exists_inner
|
||||
concurrent_completions_share_durable_bucket_quota_reservations
|
||||
multipart_completion_commits_after_quota_fleet_proof_loss_before_rename
|
||||
data_movement_multipart_completion_has_zero_quota_growth
|
||||
cancelled_multipart_completion_emits_once_and_replay_does_not_emit
|
||||
rejected_empty_parts_preserve_existing_object_and_staging
|
||||
execute_complete_multipart_upload_allows_duplicate_part_numbers_by_using_last_occurrence
|
||||
execute_complete_multipart_upload_rejects_out_of_order_parts
|
||||
validate_complete_multipart_parts_enforces_part_number_range
|
||||
normalize_complete_multipart_parts_keeps_last_duplicate_part
|
||||
execute_complete_multipart_upload_rejects_object_lock_headers
|
||||
""",
|
||||
),
|
||||
}
|
||||
|
||||
# Test module items that are not code to assign.
|
||||
TEST_MODS = ("body_read_tests", "copy_admission_tests")
|
||||
|
||||
CODE, COMMENT, LITERAL = 0, 1, 2
|
||||
IDENT_CHARS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_")
|
||||
CHAR_LITERAL = re.compile(r"'(?:\\(?:[nrt\\0'\"]|x[0-9a-fA-F]{2}|u\{[0-9a-fA-F_]{1,6}\})|[^\\'\n])'")
|
||||
RAW_STRING_START = re.compile(r'r(#*)"')
|
||||
# A relative path in code moved one module down: `super::object::..` named a sibling of
|
||||
# multipart_usecase, and from an operation file that is `super::super::object::..`.
|
||||
SUPER_PATH = re.compile(r"(?<![A-Za-z0-9_:])super::(?!\*)")
|
||||
S3S_PATH = re.compile(r"(?<![\"A-Za-z0-9_])s3s::")
|
||||
VIS = r"(?:pub(?:\([^)]*\))?\s+)?"
|
||||
FN_HEAD = re.compile(r"^" + VIS + r"(?:const\s+)?(?:async\s+)?(?:unsafe\s+)?fn\s+([A-Za-z_][A-Za-z0-9_]*)")
|
||||
STRUCT_HEAD = re.compile(r"^" + VIS + r"struct\s+([A-Za-z_][A-Za-z0-9_]*)")
|
||||
IMPL_HEAD = re.compile(r"^impl\s+([A-Za-z_][A-Za-z0-9_]*)\s*\{")
|
||||
MOD_BLOCK_HEAD = re.compile(r"^" + VIS + r"mod\s+([A-Za-z_][A-Za-z0-9_]*)\s*\{")
|
||||
MOD_DECL_HEAD = re.compile(r"^" + VIS + r"mod\s+([A-Za-z_][A-Za-z0-9_]*)\s*;")
|
||||
USE_HEAD = re.compile(r"^" + VIS + r"use\s")
|
||||
|
||||
|
||||
class SplitError(Exception):
|
||||
"""The input cannot be split as asked: exit 1."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lexing: blank comments and literals so structure can be read from the rest.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def classify(text):
|
||||
"""Return one kind byte per character: CODE, COMMENT or LITERAL."""
|
||||
kinds = bytearray(len(text))
|
||||
i, n = 0, len(text)
|
||||
|
||||
def mark(start, end, kind):
|
||||
kinds[start:end] = bytes([kind]) * (end - start)
|
||||
|
||||
while i < n:
|
||||
ch = text[i]
|
||||
if text.startswith("//", i):
|
||||
end = text.find("\n", i)
|
||||
end = n if end < 0 else end
|
||||
mark(i, end, COMMENT)
|
||||
i = end
|
||||
elif text.startswith("/*", i):
|
||||
depth, j = 1, i + 2
|
||||
while j < n and depth:
|
||||
if text.startswith("/*", j):
|
||||
depth, j = depth + 1, j + 2
|
||||
elif text.startswith("*/", j):
|
||||
depth, j = depth - 1, j + 2
|
||||
else:
|
||||
j += 1
|
||||
mark(i, j, COMMENT)
|
||||
i = j
|
||||
elif ch == "r" and (i == 0 or text[i - 1] not in IDENT_CHARS or (text[i - 1] == "b" and (i == 1 or text[i - 2] not in IDENT_CHARS))):
|
||||
raw = RAW_STRING_START.match(text, i)
|
||||
if raw is None:
|
||||
i += 1
|
||||
continue
|
||||
closing = '"' + raw.group(1)
|
||||
end = text.find(closing, raw.end())
|
||||
end = n if end < 0 else end + len(closing)
|
||||
mark(i, end, LITERAL)
|
||||
i = end
|
||||
elif ch == '"':
|
||||
j = i + 1
|
||||
while j < n and text[j] != '"':
|
||||
j += 2 if text[j] == "\\" else 1
|
||||
mark(i, min(j + 1, n), LITERAL)
|
||||
i = j + 1
|
||||
elif ch == "'":
|
||||
literal = CHAR_LITERAL.match(text, i)
|
||||
if literal is None: # a lifetime or a loop label
|
||||
i += 1
|
||||
continue
|
||||
mark(i, literal.end(), LITERAL)
|
||||
i = literal.end()
|
||||
else:
|
||||
i += 1
|
||||
return kinds
|
||||
|
||||
|
||||
def code_only(text):
|
||||
"""`text` with comments and literals replaced by spaces, newlines kept."""
|
||||
kinds = classify(text)
|
||||
return "".join(ch if kinds[i] == CODE or ch == "\n" else " " for i, ch in enumerate(text))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Item splitting. The input is rustfmt output, so an item is the run of lines
|
||||
# from its first attribute or comment to the line on which brackets balance
|
||||
# and the line ends in `}` or `;`.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Item:
|
||||
def __init__(self, lines, code_lines):
|
||||
self.lines = lines # original lines, leading comments and attributes included
|
||||
self.code_lines = code_lines # the same lines with comments and literals blanked
|
||||
self.head = next((line.strip() for line in code_lines if line.strip() and not line.strip().startswith("#[")), "")
|
||||
self.cfg_test = any(line.strip() == "#[cfg(test)]" for line in lines)
|
||||
|
||||
@property
|
||||
def text(self):
|
||||
return "\n".join(self.lines)
|
||||
|
||||
@property
|
||||
def code(self):
|
||||
return "\n".join(self.code_lines)
|
||||
|
||||
def name(self):
|
||||
for pattern in (FN_HEAD, STRUCT_HEAD, IMPL_HEAD, MOD_BLOCK_HEAD, MOD_DECL_HEAD):
|
||||
match = pattern.match(self.head)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
|
||||
def is_use(self):
|
||||
return bool(USE_HEAD.match(self.head))
|
||||
|
||||
|
||||
def split_items(lines, code_lines, what):
|
||||
"""Cut `lines` into items; blank lines separate them."""
|
||||
items, i, n = [], 0, len(lines)
|
||||
while i < n:
|
||||
if not lines[i].strip():
|
||||
i += 1
|
||||
continue
|
||||
start, balance, j = i, 0, i
|
||||
while j < n:
|
||||
stripped = code_lines[j].rstrip()
|
||||
balance += sum(stripped.count(c) for c in "([{") - sum(stripped.count(c) for c in ")]}")
|
||||
if balance == 0 and stripped.endswith(("}", ";")):
|
||||
break
|
||||
j += 1
|
||||
if j >= n:
|
||||
raise SplitError(f"{what}: an item starting at line {start + 1} never closes")
|
||||
items.append(Item(lines[start : j + 1], code_lines[start : j + 1]))
|
||||
i = j + 1
|
||||
return items
|
||||
|
||||
|
||||
def canonical(items, separator):
|
||||
return separator.join(item.text for item in items)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Use trees.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def split_top(inner):
|
||||
parts, depth, current = [], 0, []
|
||||
for ch in inner:
|
||||
if ch == "{":
|
||||
depth += 1
|
||||
elif ch == "}":
|
||||
depth -= 1
|
||||
if ch == "," and depth == 0:
|
||||
parts.append("".join(current))
|
||||
current = []
|
||||
else:
|
||||
current.append(ch)
|
||||
parts.append("".join(current))
|
||||
return [p.strip() for p in parts if p.strip()]
|
||||
|
||||
|
||||
def flatten_use(tree, prefix=""):
|
||||
"""[(full path, bound name)] for a use tree; a glob binds `*`, `as _` binds `_`."""
|
||||
tree = tree.strip()
|
||||
brace = tree.find("{")
|
||||
if brace >= 0:
|
||||
head = tree[:brace].strip()
|
||||
inner = tree[brace + 1 : tree.rindex("}")]
|
||||
base = prefix + head
|
||||
found = []
|
||||
for part in split_top(inner):
|
||||
found.extend(flatten_use(part, base))
|
||||
return found
|
||||
alias = None
|
||||
if " as " in tree:
|
||||
tree, alias = (side.strip() for side in tree.split(" as ", 1))
|
||||
path = prefix + tree
|
||||
last = path.split("::")[-1]
|
||||
if last == "self":
|
||||
path = path[: -len("::self")]
|
||||
last = path.split("::")[-1]
|
||||
return [(path, alias if alias else last)]
|
||||
|
||||
|
||||
def use_paths(item):
|
||||
body = re.sub(r"^\s*(?:#\[[^\n]*\]\s*)*", "", item.code, flags=re.S)
|
||||
body = re.sub(r"^" + VIS + r"use\s+", "", body.strip())
|
||||
body = body.rstrip().rstrip(";")
|
||||
return flatten_use(body)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The parsed input.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Parsed:
|
||||
pass
|
||||
|
||||
|
||||
def parse(text):
|
||||
lines = text.split("\n")
|
||||
if lines and lines[-1] == "":
|
||||
lines.pop()
|
||||
code_lines = code_only("\n".join(lines)).split("\n")
|
||||
|
||||
# Leading licence and `//!` lines are the file preamble.
|
||||
first = 0
|
||||
while first < len(lines) and (lines[first].startswith("//") or not lines[first].strip()):
|
||||
first += 1
|
||||
preamble = lines[:first]
|
||||
items = split_items(lines[first:], code_lines[first:], "multipart_usecase.rs")
|
||||
|
||||
parsed = Parsed()
|
||||
parsed.preamble = preamble
|
||||
parsed.imports, parsed.functions, parsed.body = [], [], []
|
||||
parsed.struct = parsed.impl = parsed.tests = None
|
||||
for item in items:
|
||||
if not item.is_use():
|
||||
parsed.body.append(item)
|
||||
name = item.name()
|
||||
if item.is_use():
|
||||
if parsed.functions or parsed.struct or parsed.impl:
|
||||
raise SplitError("a `use` item follows code: imports are expected first")
|
||||
parsed.imports.append(item)
|
||||
elif STRUCT_HEAD.match(item.head) and name == STRUCT:
|
||||
parsed.struct = item
|
||||
elif IMPL_HEAD.match(item.head) and name == STRUCT:
|
||||
if parsed.impl is not None:
|
||||
raise SplitError(f"more than one `impl {STRUCT}` block")
|
||||
parsed.impl = item
|
||||
elif MOD_BLOCK_HEAD.match(item.head) and name == "tests":
|
||||
parsed.tests = item
|
||||
elif FN_HEAD.match(item.head):
|
||||
item.after_impl = parsed.impl is not None
|
||||
parsed.functions.append(item)
|
||||
elif MOD_DECL_HEAD.match(item.head):
|
||||
raise SplitError(f"{SOURCE} already declares {item.head!r}: it is split already; restore the pre-split files first")
|
||||
else:
|
||||
raise SplitError(f"unrecognised top-level item: {item.head!r}")
|
||||
for what, value in (("struct", parsed.struct), ("impl", parsed.impl), ("tests module", parsed.tests)):
|
||||
if value is None:
|
||||
raise SplitError(f"the {what} was not found; is the input already split?")
|
||||
|
||||
# Spacing must be the canonical one, or the output would not be a move.
|
||||
rebuilt = "\n".join(preamble) + "\n" + "\n".join(i.text for i in parsed.imports) + "\n\n"
|
||||
rebuilt += canonical(parsed.body, "\n\n")
|
||||
if "\n".join(lines) != "\n".join(rebuilt.split("\n")):
|
||||
raise SplitError("the item layout is not the one this script reassembles (spacing differs from rustfmt's)")
|
||||
|
||||
parsed.impl_methods = body_items(parsed.impl, "impl")
|
||||
parsed.test_items = body_items(parsed.tests, "mod tests")
|
||||
return parsed
|
||||
|
||||
|
||||
def body_items(block, what):
|
||||
"""Items between the opening line and the closing brace of `block`."""
|
||||
opening = next(i for i, line in enumerate(block.code_lines) if line.strip() and not line.strip().startswith("#["))
|
||||
inner, inner_code = block.lines[opening + 1 : -1], block.code_lines[opening + 1 : -1]
|
||||
items = split_items(inner, inner_code, what)
|
||||
return items
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Assignment and checks.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def assign(items, table, what):
|
||||
placed = {}
|
||||
seen = set()
|
||||
for item in items:
|
||||
name = item.name()
|
||||
if name is None:
|
||||
raise SplitError(f"{what}: cannot name the item {item.head!r}")
|
||||
if name in seen:
|
||||
raise SplitError(f"{what}: two items are named {name}")
|
||||
seen.add(name)
|
||||
target = table.get(name)
|
||||
if target is None:
|
||||
raise SplitError(f"{what}: no assignment for {name} ({item.head[:60]!r})")
|
||||
placed[name] = (target, item)
|
||||
unused = sorted(set(table) - seen)
|
||||
if unused:
|
||||
raise SplitError(f"{what}: assignments that match nothing: {', '.join(unused)}")
|
||||
return placed
|
||||
|
||||
|
||||
def check_targets(table, what, allowed):
|
||||
for name, target in table.items():
|
||||
if target not in allowed:
|
||||
raise SplitError(f"{what}: {name} is assigned to unknown file {target!r}")
|
||||
|
||||
|
||||
def word(name):
|
||||
return re.compile(r"(?<![A-Za-z0-9_])" + re.escape(name) + r"(?![A-Za-z0-9_])")
|
||||
|
||||
|
||||
def referenced_from(name, texts, own):
|
||||
"""Files other than `own` whose code mentions `name`."""
|
||||
pattern = word(name)
|
||||
return sorted(f for f, code in texts.items() if f != own and pattern.search(code))
|
||||
|
||||
|
||||
def plan(parsed):
|
||||
"""Decide the content of every output file; returns (pieces, report)."""
|
||||
check_targets(FUNCTIONS, "functions", {PARENT, *OPS})
|
||||
check_targets(METHODS, "methods", {PARENT, *OPS})
|
||||
check_targets(TESTS, "tests", {PARENT, SUPPORT, *OPS})
|
||||
functions = assign(parsed.functions, FUNCTIONS, "functions")
|
||||
methods = assign(parsed.impl_methods, METHODS, "methods")
|
||||
|
||||
test_code_items = []
|
||||
test_uses, test_mods, test_glob = [], [], None
|
||||
for item in parsed.test_items:
|
||||
if item.is_use():
|
||||
if item.head.replace(" ", "") == "usesuper::*;":
|
||||
test_glob = item
|
||||
else:
|
||||
test_uses.append(item)
|
||||
elif MOD_DECL_HEAD.match(item.head):
|
||||
if item.name() not in TEST_MODS:
|
||||
raise SplitError(f"mod tests: unexpected module declaration {item.head!r}")
|
||||
test_mods.append(item)
|
||||
else:
|
||||
test_code_items.append(item)
|
||||
if test_glob is None:
|
||||
raise SplitError("mod tests does not start with `use super::*;`")
|
||||
if sorted(m.name() for m in test_mods) != sorted(TEST_MODS):
|
||||
raise SplitError("mod tests: the declared test modules differ from TEST_MODS")
|
||||
tests = assign(test_code_items, TESTS, "tests")
|
||||
|
||||
for placed in (functions, methods):
|
||||
for target, item in placed.values():
|
||||
if target != PARENT:
|
||||
push_down(item)
|
||||
|
||||
# Piece lists per target file.
|
||||
pieces = {f: {"functions": [], "methods": [], "tests": []} for f in (PARENT, SUPPORT, *OPS)}
|
||||
for _, (target, item) in functions.items():
|
||||
pieces[target]["functions"].append(item)
|
||||
for _, (target, item) in methods.items():
|
||||
pieces[target]["methods"].append(item)
|
||||
for _, (target, item) in tests.items():
|
||||
pieces[target]["tests"].append(item)
|
||||
if pieces[SUPPORT]["functions"] or pieces[SUPPORT]["methods"]:
|
||||
raise SplitError("test_support holds tests-module fixtures only")
|
||||
pieces["_uses"] = test_uses
|
||||
pieces["_mods"] = test_mods
|
||||
return pieces, (functions, methods, tests)
|
||||
|
||||
|
||||
def per_file_code(parsed, pieces, extra):
|
||||
"""Code of each output file for the reference check (parent keeps imports)."""
|
||||
texts = {}
|
||||
for target in (PARENT, SUPPORT, *OPS):
|
||||
parts = []
|
||||
if target == PARENT:
|
||||
parts += [i.code for i in parsed.imports] + [parsed.struct.code]
|
||||
parts += [i.code for i in pieces[target]["functions"]]
|
||||
parts += [i.code for i in pieces[target]["methods"]]
|
||||
parts += [i.code for i in pieces[target]["tests"]]
|
||||
texts[target] = "\n".join(parts)
|
||||
for target, code in extra.items():
|
||||
texts[target] += "\n" + code
|
||||
return texts
|
||||
|
||||
|
||||
def reference_check(pieces, texts):
|
||||
"""A private item moved out of the parent must stay inside its own file.
|
||||
|
||||
Items kept in the parent and fixtures in test_support are visible to every
|
||||
child through the globs, so only items placed in an operation file are
|
||||
checked. A name that is also a local binding elsewhere would show up here as
|
||||
a false alarm; the message names the files so the table can be adjusted.
|
||||
"""
|
||||
problems = []
|
||||
for group in ("functions", "methods", "tests"):
|
||||
for target in OPS:
|
||||
for item in pieces[target][group]:
|
||||
name = item.name()
|
||||
if name is None or not FN_HEAD.match(item.head):
|
||||
continue
|
||||
if re.match(r"^pub(\(|\s)", item.head) and group == "methods":
|
||||
continue # a public method is reached through the type
|
||||
users = referenced_from(name, texts, target)
|
||||
if users:
|
||||
problems.append(f"{name} is placed in {target} but also mentioned in {', '.join(users)}")
|
||||
return problems
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rendering.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def rewrite_code(text, pattern, replacement):
|
||||
"""Replace `pattern` matches that sit in code, never in comments or literals."""
|
||||
kinds = classify(text)
|
||||
out, last = [], 0
|
||||
for match in pattern.finditer(text):
|
||||
if kinds[match.start()] != CODE:
|
||||
continue
|
||||
out.append(text[last : match.start()])
|
||||
out.append(replacement)
|
||||
last = match.end()
|
||||
out.append(text[last:])
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def rewrite_s3s(text):
|
||||
"""Send code-position `s3s::` paths through the sanctioned funnel."""
|
||||
return rewrite_code(text, S3S_PATH, FUNNEL)
|
||||
|
||||
|
||||
def push_down(item):
|
||||
"""Make the relative paths of an item that leaves the parent module point where they did."""
|
||||
text = rewrite_code(item.text, SUPER_PATH, "super::super::")
|
||||
item.lines = text.split("\n")
|
||||
item.code_lines = code_only(text).split("\n")
|
||||
|
||||
|
||||
def licence(preamble):
|
||||
block = []
|
||||
for line in preamble:
|
||||
if not line.startswith("//") or line.startswith("//!"):
|
||||
break
|
||||
block.append(line)
|
||||
return "\n".join(block)
|
||||
|
||||
|
||||
def doc_line(preamble):
|
||||
return next(line for line in preamble if line.startswith("//!"))
|
||||
|
||||
|
||||
def join_items(items):
|
||||
return "\n\n".join(item.text for item in items)
|
||||
|
||||
|
||||
def render_operation(op, pieces, header, tests_extra_mods):
|
||||
out = [header, f"//! {OP_DOC[op]}", "", "use super::*;", ""]
|
||||
body = []
|
||||
if pieces[op]["functions"]:
|
||||
body.append(join_items(pieces[op]["functions"]))
|
||||
if pieces[op]["methods"]:
|
||||
body.append(f"impl {STRUCT} {{\n{join_items(pieces[op]['methods'])}\n}}")
|
||||
if pieces[op]["tests"] or tests_extra_mods:
|
||||
inner = [join_items(pieces[op]["tests"])] if pieces[op]["tests"] else []
|
||||
inner += [f" mod {name};" for name in tests_extra_mods]
|
||||
body.append(
|
||||
"#[cfg(test)]\nmod tests {\n use super::super::test_support::*;\n use super::*;\n\n"
|
||||
+ "\n\n".join(inner)
|
||||
+ "\n}"
|
||||
)
|
||||
out.append("\n\n".join(body))
|
||||
return rewrite_s3s("\n".join(out) + "\n")
|
||||
|
||||
|
||||
def render_test_support(parsed, pieces, header, parent_bound):
|
||||
out = [header, "//! Fixtures and test-only imports shared by the multipart use-case tests.", "", "use super::*;"]
|
||||
reexports, notes = [], []
|
||||
for item in pieces["_uses"]:
|
||||
for path, name in use_paths(item):
|
||||
if name in parent_bound:
|
||||
notes.append((name, parent_bound[name], path))
|
||||
continue
|
||||
reexports.append(f"pub(super) use {path};")
|
||||
out.extend(reexports)
|
||||
out.append("")
|
||||
fixtures = []
|
||||
for item in pieces[SUPPORT]["tests"]:
|
||||
lines = list(item.lines)
|
||||
for index, line in enumerate(lines):
|
||||
if FN_HEAD.match(line.strip()) and not line.startswith(" #"):
|
||||
lines[index] = re.sub(r"^(\s*)", r"\1pub(super) ", line, count=1)
|
||||
break
|
||||
fixtures.append("\n".join(line[4:] if line.startswith(" ") else line for line in lines))
|
||||
out.append("\n\n".join(fixtures))
|
||||
return rewrite_s3s("\n".join(out) + "\n"), notes
|
||||
|
||||
|
||||
def render_parent(parsed, pieces, header):
|
||||
out = [header, "\n".join(line for line in parsed.preamble if line.startswith("//!")), ""]
|
||||
out.append("\n".join(i.text for i in parsed.imports))
|
||||
out.append("")
|
||||
mods = [f"mod {op};" for op in sorted(OPS + (SUPPORT,))]
|
||||
mods = [("#[cfg(test)]\n" + m) if SUPPORT in m else m for m in mods]
|
||||
out.append("\n".join(mods))
|
||||
out.append("")
|
||||
before = [i for i in pieces[PARENT]["functions"] if not i.after_impl]
|
||||
after = [i for i in pieces[PARENT]["functions"] if i.after_impl]
|
||||
chunks = [join_items(before)] if before else []
|
||||
chunks.append(
|
||||
f"{parsed.struct.text}\n\n{parsed.impl.lines[0]}\n{join_items(pieces[PARENT]['methods'])}\n}}"
|
||||
)
|
||||
if after:
|
||||
chunks.append(join_items(after))
|
||||
if pieces[PARENT]["tests"]:
|
||||
chunks.append(
|
||||
rewrite_s3s(
|
||||
"#[cfg(test)]\nmod tests {\n use super::test_support::*;\n use super::*;\n\n"
|
||||
+ join_items(pieces[PARENT]["tests"])
|
||||
+ "\n}"
|
||||
)
|
||||
)
|
||||
out.append("\n\n".join(chunks))
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Driver.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def bound_names(imports):
|
||||
bound = {}
|
||||
for item in imports:
|
||||
for path, name in use_paths(item):
|
||||
if name not in ("*", "_"):
|
||||
bound[name] = path
|
||||
return bound
|
||||
|
||||
|
||||
def build(root):
|
||||
source_path = root / SOURCE
|
||||
if not source_path.is_file():
|
||||
raise SplitError(f"{SOURCE} is missing")
|
||||
text = source_path.read_text(encoding="utf-8")
|
||||
parsed = parse(text)
|
||||
pieces, (functions, methods, tests) = plan(parsed)
|
||||
|
||||
extra = {}
|
||||
old_texts = {}
|
||||
for name, (path, op) in OLD_TEST_FILES.items():
|
||||
file = root / path
|
||||
if not file.is_file():
|
||||
raise SplitError(f"{path} is missing")
|
||||
old_texts[name] = file.read_text(encoding="utf-8")
|
||||
extra[op] = extra.get(op, "") + "\n" + code_only(old_texts[name])
|
||||
texts = per_file_code(parsed, pieces, extra)
|
||||
problems = reference_check(pieces, texts)
|
||||
if problems:
|
||||
raise SplitError("cross-file references:\n " + "\n ".join(problems))
|
||||
|
||||
header = licence(parsed.preamble) + "\n"
|
||||
files = {}
|
||||
parent_bound = bound_names(parsed.imports)
|
||||
support, notes = render_test_support(parsed, pieces, header, parent_bound)
|
||||
files[f"{OUT_DIR}/test_support.rs"] = support
|
||||
for op in OPS:
|
||||
mods = [name for name, (_, owner) in OLD_TEST_FILES.items() if owner == op]
|
||||
files[f"{OUT_DIR}/{op}.rs"] = render_operation(op, pieces, header, mods)
|
||||
for name, (path, op) in OLD_TEST_FILES.items():
|
||||
files[f"{OUT_DIR}/{op}/tests/{name}.rs"] = old_texts[name]
|
||||
files[SOURCE] = render_parent(parsed, pieces, header)
|
||||
return parsed, pieces, files, notes, (functions, methods, tests)
|
||||
|
||||
|
||||
def check_footprint(files):
|
||||
for path, text in files.items():
|
||||
if path == SOURCE:
|
||||
continue
|
||||
for number, line in enumerate(text.split("\n"), 1):
|
||||
if S3S_PATH.search(line):
|
||||
raise SplitError(f"{path}:{number} still names s3s directly: {line.strip()}")
|
||||
return None
|
||||
|
||||
|
||||
FUNNEL_NAMES = {"dto", "xml", "S3", "Body", "S3Request", "S3ErrorCode", "S3Response", "S3Error", "S3Result"}
|
||||
|
||||
|
||||
def check_funnel_names(parsed_sources, files):
|
||||
seen = set()
|
||||
original = "\n".join(parsed_sources)
|
||||
for match in S3S_PATH.finditer(original):
|
||||
tail = re.match(r"[A-Za-z_][A-Za-z0-9_]*", original[match.end() :])
|
||||
if tail:
|
||||
seen.add(tail.group(0))
|
||||
unknown = sorted(seen - FUNNEL_NAMES)
|
||||
if unknown:
|
||||
raise SplitError(f"s3s paths with no funnel re-export: {', '.join(unknown)}")
|
||||
|
||||
|
||||
def edit_storage_api(text):
|
||||
if "pub(crate) use s3s::dto;" in text:
|
||||
raise SplitError(f"{STORAGE_API} already re-exports s3s::dto")
|
||||
anchor = " #[cfg(test)]\n pub(crate) use s3s::dto::{GetObjectInput, HeadObjectInput, ListObjectsV2Input, ListObjectsV2Output};\n"
|
||||
if text.count(anchor) != 1:
|
||||
raise SplitError(f"{STORAGE_API}: the s3 module's dto re-export anchor was not found exactly once")
|
||||
text = text.replace(anchor, " pub(crate) use s3s::dto;\n" + anchor)
|
||||
anchor = " #[cfg(test)]\n pub(crate) use s3s::xml::{Serialize as XmlSerialize, Serializer as XmlSerializer};\n"
|
||||
if text.count(anchor) != 1:
|
||||
raise SplitError(f"{STORAGE_API}: the s3 module's xml re-export anchor was not found exactly once")
|
||||
return text.replace(anchor, " #[cfg(test)]\n pub(crate) use s3s::xml;\n" + anchor)
|
||||
|
||||
|
||||
def edit_nextest(text, tests):
|
||||
"""Point the filter that pins one test at the module the test now lives in."""
|
||||
replaced = 0
|
||||
for name, (target, _) in tests.items():
|
||||
needle = f"app::multipart_usecase::tests::{name}"
|
||||
if needle in text:
|
||||
moved = "app::multipart_usecase::tests" if target == PARENT else f"app::multipart_usecase::{target}::tests"
|
||||
replaced += text.count(needle)
|
||||
text = text.replace(needle, f"{moved}::{name}")
|
||||
if replaced == 0:
|
||||
raise SplitError(f"{NEXTEST} names no multipart_usecase test: has the filter moved?")
|
||||
return text
|
||||
|
||||
|
||||
def run(root, fmt, report):
|
||||
parsed, pieces, files, notes, (functions, methods, tests) = build(root)
|
||||
check_funnel_names([i.text for i in parsed.functions + parsed.impl_methods + parsed.test_items], files)
|
||||
check_footprint(files)
|
||||
if report:
|
||||
print_report(parsed, pieces, files, notes, (functions, methods, tests))
|
||||
return 0
|
||||
|
||||
storage = edit_storage_api((root / STORAGE_API).read_text(encoding="utf-8"))
|
||||
nextest = edit_nextest((root / NEXTEST).read_text(encoding="utf-8"), tests)
|
||||
|
||||
for path, text in files.items():
|
||||
target = root / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(text, encoding="utf-8")
|
||||
for name, (path, op) in OLD_TEST_FILES.items():
|
||||
(root / path).unlink()
|
||||
old_dir = root / OUT_DIR / "tests"
|
||||
if old_dir.is_dir() and not any(old_dir.iterdir()):
|
||||
old_dir.rmdir()
|
||||
(root / STORAGE_API).write_text(storage, encoding="utf-8")
|
||||
(root / NEXTEST).write_text(nextest, encoding="utf-8")
|
||||
|
||||
if fmt:
|
||||
written = [str(root / p) for p in files if p != SOURCE] + [str(root / SOURCE), str(root / STORAGE_API)]
|
||||
result = subprocess.run(["rustfmt", "--edition", "2024", *written], check=False, cwd=root)
|
||||
if result.returncode != 0:
|
||||
raise SplitError("rustfmt failed on the written files")
|
||||
summary(files, pieces)
|
||||
return 0
|
||||
|
||||
|
||||
def print_report(parsed, pieces, files, notes, assigned):
|
||||
for target in (PARENT, SUPPORT, *OPS):
|
||||
counts = {k: len(pieces[target][k]) for k in ("functions", "methods", "tests")}
|
||||
print(f"{target}: {counts}")
|
||||
for name, parent_path, test_path in notes:
|
||||
print(f"note: test import {name} is already bound in the parent ({parent_path}); test path {test_path}")
|
||||
for path, text in sorted(files.items()):
|
||||
print(f"{path}: {len(text.splitlines())} lines")
|
||||
|
||||
|
||||
def summary(files, pieces):
|
||||
for path, text in sorted(files.items()):
|
||||
print(f"wrote {path}: {len(text.splitlines())} lines")
|
||||
for target in (PARENT, SUPPORT, *OPS):
|
||||
fns = len(pieces[target]["functions"]) + len(pieces[target]["methods"])
|
||||
tests = sum(1 for i in pieces[target]["tests"] if re.search(r"#\[(tokio::)?test", i.text))
|
||||
print(f"{target}: {fns} production fns, {tests} tests")
|
||||
|
||||
|
||||
def main(argv):
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0])
|
||||
parser.add_argument("--no-fmt", action="store_true", help="do not run rustfmt on the written files")
|
||||
parser.add_argument("--report", action="store_true", help="print the assignment and write nothing")
|
||||
parser.add_argument("--self-test", action="store_true", help="run the script's own tests")
|
||||
args = parser.parse_args(argv)
|
||||
if args.self_test:
|
||||
suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTest)
|
||||
return 0 if unittest.TextTestRunner(verbosity=1).run(suite).wasSuccessful() else 1
|
||||
try:
|
||||
return run(ROOT, not args.no_fmt, args.report)
|
||||
except SplitError as error:
|
||||
print(f"split_multipart_usecase: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
class SelfTest(unittest.TestCase):
|
||||
SAMPLE = "\n".join(
|
||||
[
|
||||
"// licence",
|
||||
"",
|
||||
"//! doc",
|
||||
"",
|
||||
"use a::b;",
|
||||
"",
|
||||
"/// doc",
|
||||
"fn one(",
|
||||
" x: u8,",
|
||||
") -> u8 {",
|
||||
' let s = "}";',
|
||||
" x",
|
||||
"}",
|
||||
"",
|
||||
"#[cfg(test)]",
|
||||
"mod tests {",
|
||||
" use super::*;",
|
||||
"",
|
||||
" #[test]",
|
||||
" fn t() {}",
|
||||
"}",
|
||||
]
|
||||
)
|
||||
|
||||
def test_items_split_on_balance_not_on_braces_in_strings(self):
|
||||
lines = self.SAMPLE.split("\n")[6:]
|
||||
items = split_items(lines, code_only("\n".join(lines)).split("\n"), "sample")
|
||||
self.assertEqual([i.name() for i in items], ["one", "tests"])
|
||||
self.assertTrue(items[1].cfg_test)
|
||||
|
||||
def test_use_tree_flattening(self):
|
||||
paths = flatten_use("a::{b, c::{d as e, f}, g::*}")
|
||||
self.assertEqual(paths, [("a::b", "b"), ("a::c::d", "e"), ("a::c::f", "f"), ("a::g::*", "*")])
|
||||
|
||||
def test_s3s_rewrite_skips_strings_and_identifiers(self):
|
||||
text = 'let a = s3s::dto::X; let b = "s3s::dto"; let c = rs3s::y;'
|
||||
self.assertEqual(rewrite_s3s(text), f'let a = {FUNNEL}dto::X; let b = "s3s::dto"; let c = rs3s::y;')
|
||||
|
||||
def test_unassigned_item_fails(self):
|
||||
lines = self.SAMPLE.split("\n")[6:13]
|
||||
items = split_items(lines, code_only("\n".join(lines)).split("\n"), "sample")
|
||||
with self.assertRaises(SplitError):
|
||||
assign(items, {}, "functions")
|
||||
with self.assertRaises(SplitError):
|
||||
assign(items, {"one": PARENT, "ghost": PARENT}, "functions")
|
||||
|
||||
def test_unclosed_item_fails(self):
|
||||
with self.assertRaises(SplitError):
|
||||
split_items(["fn a() {"], ["fn a() {"], "sample")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Source-set helper for text guards that must survive a mechanical file split.
|
||||
# Sourced by check_architecture_migration_rules.sh and check_logging_guardrails.sh;
|
||||
# contract tests: scripts/test_guard_source_set.sh.
|
||||
#
|
||||
# A guard that greps one named file stops seeing the code it guards once that
|
||||
# file is split into a sibling directory. guard_source_set resolves the set of
|
||||
# files such a guard is about instead:
|
||||
#
|
||||
# guard_source_set <root> <required-file> [<split-dir>...]
|
||||
#
|
||||
# On success it sets the global array GUARD_SOURCES to <required-file> followed
|
||||
# by every `.rs` file under each <split-dir> that exists, in byte order of path
|
||||
# (LC_ALL=C), all relative to <root>. An absent <split-dir> contributes nothing.
|
||||
# It returns 1 with a message on stderr when <required-file> is not a regular
|
||||
# file (a guard whose input is missing must fail, not pass) or when a
|
||||
# <split-dir> exists but is not a directory. On failure GUARD_SOURCES still
|
||||
# holds <required-file>, so a caller that records the failure and continues
|
||||
# hands the missing path to its search tool rather than nothing.
|
||||
|
||||
guard_source_set() {
|
||||
local root="$1" required="$2" split_dir path listing
|
||||
shift 2
|
||||
GUARD_SOURCES=("$required")
|
||||
|
||||
if [[ ! -f "${root}/${required}" ]]; then
|
||||
printf 'guard source %s is missing\n' "$required" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
for split_dir in "$@"; do
|
||||
if [[ ! -e "${root}/${split_dir}" ]]; then
|
||||
continue
|
||||
fi
|
||||
if [[ ! -d "${root}/${split_dir}" ]]; then
|
||||
printf 'guard split path %s is not a directory\n' "$split_dir" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! listing="$(set -o pipefail; cd "$root" && find "$split_dir" -type f -name '*.rs' | LC_ALL=C sort)"; then
|
||||
printf 'guard split directory %s cannot be listed\n' "$split_dir" >&2
|
||||
return 1
|
||||
fi
|
||||
while IFS= read -r path; do
|
||||
if [[ -n "$path" ]]; then
|
||||
GUARD_SOURCES+=("$path")
|
||||
fi
|
||||
done <<<"$listing"
|
||||
done
|
||||
return 0
|
||||
}
|
||||
@@ -509,7 +509,9 @@ pass/fail table in the job summary.
|
||||
incomplete parameterized cases.
|
||||
- `api_coverage.py` — quantifies S3 API surface coverage by comparing the
|
||||
s3s `S3` trait (at the revision pinned in Cargo.toml) against the methods
|
||||
RustFS overrides in `impl S3 for FS`:
|
||||
RustFS overrides in `impl S3 for FS`, found exactly once across
|
||||
`rustfs/src/storage/ecfs.rs` and `rustfs/src/storage/legacy_edge/`
|
||||
(contract tests: `test_api_coverage.py`, run by `make script-tests`):
|
||||
|
||||
```bash
|
||||
python3 scripts/s3-tests/api_coverage.py # summary + missing ops
|
||||
|
||||
@@ -17,7 +17,8 @@
|
||||
|
||||
Compares the operations defined by the s3s `S3` trait (at the revision pinned
|
||||
in the workspace Cargo.toml) against the methods RustFS overrides in
|
||||
`impl S3 for FS` (rustfs/src/storage/ecfs.rs). Trait methods that are not
|
||||
`impl S3 for FS` (rustfs/src/storage/ecfs.rs, or the rustfs/src/storage/legacy_edge/
|
||||
directory a mechanical split of that file moves code into). Trait methods that are not
|
||||
overridden fall back to the s3s default implementation, which returns
|
||||
NotImplemented — so "overridden" is a faithful proxy for "implemented".
|
||||
|
||||
@@ -36,6 +37,8 @@ import sys
|
||||
import tempfile
|
||||
|
||||
ECFS_PATH = "rustfs/src/storage/ecfs.rs"
|
||||
ECFS_SPLIT_DIR = "rustfs/src/storage/legacy_edge"
|
||||
IMPL_MARKER = "impl S3 for FS {"
|
||||
TRAIT_RELPATH = "crates/s3s/src/s3_trait.rs"
|
||||
|
||||
|
||||
@@ -97,14 +100,28 @@ def trait_operations(trait_source: pathlib.Path) -> dict[str, str]:
|
||||
return ops
|
||||
|
||||
|
||||
def implemented_operations(ecfs_source: pathlib.Path) -> set[str]:
|
||||
src = ecfs_source.read_text(encoding="utf-8")
|
||||
marker = "impl S3 for FS {"
|
||||
start = src.find(marker)
|
||||
if start < 0:
|
||||
raise RuntimeError(f"`{marker}` not found in {ecfs_source}")
|
||||
block = extract_block(src, start)
|
||||
return set(re.findall(r"async fn (\w+)", block))
|
||||
def ecfs_sources(root: pathlib.Path) -> list[pathlib.Path]:
|
||||
"""Return ecfs.rs (required) followed by every .rs file under the split directory, in path order."""
|
||||
required = root / ECFS_PATH
|
||||
if not required.is_file():
|
||||
raise RuntimeError(f"{required} is missing")
|
||||
split_dir = root / ECFS_SPLIT_DIR
|
||||
if not split_dir.exists():
|
||||
return [required]
|
||||
if not split_dir.is_dir():
|
||||
raise RuntimeError(f"{split_dir} is not a directory")
|
||||
return [required, *sorted(split_dir.rglob("*.rs"), key=lambda path: path.relative_to(root).as_posix())]
|
||||
|
||||
|
||||
def implemented_operations(root: pathlib.Path) -> set[str]:
|
||||
"""Return the methods of the one `impl S3 for FS` block across the ECFS source set."""
|
||||
blocks = []
|
||||
for source in ecfs_sources(root):
|
||||
src = source.read_text(encoding="utf-8")
|
||||
blocks += [extract_block(src, match.start()) for match in re.finditer(re.escape(IMPL_MARKER), src)]
|
||||
if len(blocks) != 1:
|
||||
raise RuntimeError(f"`{IMPL_MARKER}` must occur exactly once under {ECFS_PATH} and {ECFS_SPLIT_DIR}, found {len(blocks)}")
|
||||
return set(re.findall(r"async fn (\w+)", blocks[0]))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -122,7 +139,7 @@ def main() -> int:
|
||||
trait_source = args.s3s_trait or locate_trait_source(git_url, rev)
|
||||
|
||||
all_ops = trait_operations(trait_source)
|
||||
impl_ops = implemented_operations(root / ECFS_PATH)
|
||||
impl_ops = implemented_operations(root)
|
||||
|
||||
stale = sorted(impl_ops - all_ops.keys())
|
||||
implemented = sorted(op for op in all_ops if op in impl_ops)
|
||||
|
||||
Executable
+93
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Contract tests for how api_coverage.py finds `impl S3 for FS` across the ECFS source set."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import api_coverage # noqa: E402
|
||||
|
||||
IMPL = "impl S3 for FS {\n async fn get_object(&self) {}\n async fn put_object(&self) {}\n}\n"
|
||||
|
||||
|
||||
class ImplementedOperationsTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmp.name)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def write(self, relative: str, text: str) -> None:
|
||||
path = self.root / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
def test_reads_the_impl_from_ecfs_when_the_split_directory_is_absent(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, "struct FS;\n" + IMPL)
|
||||
|
||||
self.assertEqual(api_coverage.implemented_operations(self.root), {"get_object", "put_object"})
|
||||
|
||||
def test_finds_the_impl_after_it_moves_into_the_split_directory(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, "struct FS;\n")
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/s3_impl.rs", IMPL)
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/bucket_core.rs", "impl FS {\n async fn helper(&self) {}\n}\n")
|
||||
|
||||
self.assertEqual(api_coverage.implemented_operations(self.root), {"get_object", "put_object"})
|
||||
|
||||
def test_missing_ecfs_fails_even_when_the_split_directory_holds_the_impl(self) -> None:
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/s3_impl.rs", IMPL)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "ecfs.rs is missing"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
def test_an_impl_in_both_ecfs_and_the_split_directory_fails(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, IMPL)
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/s3_impl.rs", IMPL)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "exactly once.*found 2"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
def test_no_impl_anywhere_fails(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, "struct FS;\n")
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/bucket_core.rs", "impl FS {}\n")
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "found 0"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
def test_an_impl_in_a_non_rust_file_of_the_split_directory_is_not_source(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, "struct FS;\n")
|
||||
self.write(f"{api_coverage.ECFS_SPLIT_DIR}/notes.md", IMPL)
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "found 0"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
def test_a_split_path_that_is_a_file_fails(self) -> None:
|
||||
self.write(api_coverage.ECFS_PATH, IMPL)
|
||||
self.write(api_coverage.ECFS_SPLIT_DIR, "stray\n")
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "not a directory"):
|
||||
api_coverage.implemented_operations(self.root)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env bash
|
||||
# Contract tests for scripts/guard_source_set.sh, the source-set helper that
|
||||
# lets text guards read a file plus the directory a mechanical split moves its
|
||||
# code into (rustfs/backlog#2765).
|
||||
#
|
||||
# Each case builds a throwaway tree; nothing in the real workspace is read.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
# shellcheck source=scripts/guard_source_set.sh
|
||||
source "${GUARD_SOURCE_SET_SCRIPT:-$SCRIPT_DIR/guard_source_set.sh}"
|
||||
TMP_ROOT="$(mktemp -d)"
|
||||
trap 'chmod -R u+rwx "$TMP_ROOT" 2>/dev/null || true; rm -rf "$TMP_ROOT"' EXIT
|
||||
|
||||
passed=0
|
||||
skipped=0
|
||||
case_no=0
|
||||
FX=""
|
||||
STATUS=0
|
||||
ERR=""
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
new_case() {
|
||||
case_no=$((case_no + 1))
|
||||
FX="$TMP_ROOT/case$case_no"
|
||||
mkdir -p "$FX"
|
||||
}
|
||||
|
||||
put() {
|
||||
mkdir -p "$(dirname "$FX/$1")"
|
||||
printf 'fn guarded() {}\n' >"$FX/$1"
|
||||
}
|
||||
|
||||
# run_set <args...>: call guard_source_set on the case tree, capturing status and stderr.
|
||||
run_set() {
|
||||
local err_file="$FX/.stderr"
|
||||
STATUS=0
|
||||
guard_source_set "$FX" "$@" 2>"$err_file" || STATUS=$?
|
||||
ERR="$(cat "$err_file")"
|
||||
}
|
||||
|
||||
sources_are() {
|
||||
local expected actual
|
||||
expected="$(printf '%s\n' "$@")"
|
||||
actual="$(printf '%s\n' "${GUARD_SOURCES[@]}")"
|
||||
[[ "$actual" == "$expected" ]] || fail "case $case_no: sources were [$actual], expected [$expected]"
|
||||
}
|
||||
|
||||
ok() {
|
||||
passed=$((passed + 1))
|
||||
}
|
||||
|
||||
# --- positive cases ---------------------------------------------------------
|
||||
|
||||
# An absent split directory contributes nothing.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 0 ]] || fail "absent split dir: status $STATUS ($ERR)"
|
||||
sources_are src/usecase.rs
|
||||
ok
|
||||
|
||||
# A present split directory adds its .rs files, recursively, in byte order,
|
||||
# after the required file; non-.rs files are not source.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
put src/usecase/zeta.rs
|
||||
put src/usecase/B.rs
|
||||
put src/usecase/a.rs
|
||||
put src/usecase/tests/nested.rs
|
||||
printf 'not rust\n' >"$FX/src/usecase/notes.md"
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 0 ]] || fail "present split dir: status $STATUS ($ERR)"
|
||||
sources_are src/usecase.rs src/usecase/B.rs src/usecase/a.rs src/usecase/tests/nested.rs src/usecase/zeta.rs
|
||||
ok
|
||||
|
||||
# Several split directories are read in argument order; an empty one adds nothing.
|
||||
new_case
|
||||
put src/ecfs.rs
|
||||
put src/legacy_edge/b.rs
|
||||
mkdir -p "$FX/src/empty"
|
||||
put src/other/a.rs
|
||||
run_set src/ecfs.rs src/legacy_edge src/empty src/missing src/other
|
||||
[[ "$STATUS" == 0 ]] || fail "several split dirs: status $STATUS ($ERR)"
|
||||
sources_are src/ecfs.rs src/legacy_edge/b.rs src/other/a.rs
|
||||
ok
|
||||
|
||||
# --- negative cases ---------------------------------------------------------
|
||||
|
||||
# A missing required file fails and names the file.
|
||||
new_case
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "missing required file: status $STATUS"
|
||||
[[ "$ERR" == *"src/usecase.rs"* ]] || fail "missing required file: stderr does not name it: $ERR"
|
||||
sources_are src/usecase.rs
|
||||
ok
|
||||
|
||||
# A missing required file fails even when the split directory holds the code:
|
||||
# the split must keep the file, and a guard must not pass on half its input.
|
||||
new_case
|
||||
put src/usecase/moved.rs
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "missing required file with split dir: status $STATUS"
|
||||
[[ "$ERR" == *"is missing"* ]] || fail "missing required file with split dir: $ERR"
|
||||
ok
|
||||
|
||||
# A required path that is a directory is not a source file.
|
||||
new_case
|
||||
mkdir -p "$FX/src/usecase.rs"
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "required directory: status $STATUS"
|
||||
ok
|
||||
|
||||
# A split path that exists as a regular file fails rather than being skipped.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
printf 'stray\n' >"$FX/src/usecase"
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "split path is a file: status $STATUS"
|
||||
[[ "$ERR" == *"not a directory"* ]] || fail "split path is a file: $ERR"
|
||||
ok
|
||||
|
||||
# A later split path that is a file still fails after an earlier valid one.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
put src/usecase/a.rs
|
||||
printf 'stray\n' >"$FX/src/second"
|
||||
run_set src/usecase.rs src/usecase src/second
|
||||
[[ "$STATUS" == 1 ]] || fail "second split path is a file: status $STATUS"
|
||||
ok
|
||||
|
||||
# An unlistable split directory fails rather than contributing nothing.
|
||||
new_case
|
||||
put src/usecase.rs
|
||||
put src/usecase/hidden/a.rs
|
||||
chmod 000 "$FX/src/usecase/hidden"
|
||||
if [[ "$(id -u)" == 0 ]]; then
|
||||
echo "SKIP: unlistable split directory case needs a non-root user (root ignores directory permissions)"
|
||||
skipped=$((skipped + 1))
|
||||
else
|
||||
run_set src/usecase.rs src/usecase
|
||||
[[ "$STATUS" == 1 ]] || fail "unlistable split dir: status $STATUS"
|
||||
[[ "$ERR" == *"cannot be listed"* ]] || fail "unlistable split dir: $ERR"
|
||||
ok
|
||||
fi
|
||||
chmod 755 "$FX/src/usecase/hidden"
|
||||
|
||||
echo "guard_source_set contract tests: $passed passed, $skipped skipped"
|
||||
Reference in New Issue
Block a user