mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-11 12:43:54 +00:00
Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1fc1982976 | |||
| e2a927a301 | |||
| ff84f2b5bf | |||
| def5418168 | |||
| 85f5cc9ead | |||
| 613cfac753 | |||
| 75292c6224 | |||
| 9d513cffe9 | |||
| c5afd3738c | |||
| 6b44869b4f |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=0374e6b9b061dd5d622686301386232a4aa1cb22eedc18af70d2a7411bc66f8d
|
||||
sha256-linux=70d8e59d9ba0512859bf1de0cb255b27b1bcf9f2d10cf12442762a025887397d
|
||||
sha256-linux=f2cee240f7a744ae991971820f48a6c9c28d19f525e8b2345fcb76eeaaf1e382
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=ac8f0faa06d836a37480e290496a46707915b289616416f61982ccd6af2601dc
|
||||
sha256=21e9a1561ac20382503c38353bbb9b830658534a4f277c254a42d0afc6ebed8b
|
||||
|
||||
@@ -70,11 +70,6 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
||||
@echo "📦 Checking s3s footprint ratchet..."
|
||||
./scripts/check_s3s_footprint.sh
|
||||
|
||||
.PHONY: gateway-stack-release-boundary-check
|
||||
gateway-stack-release-boundary-check: ## Check no release build can enable the gateway-stack feature
|
||||
@echo "🚧 Checking gateway-stack release boundary..."
|
||||
./scripts/check_gateway_stack_release_boundary.sh
|
||||
|
||||
.PHONY: test-count-ratchet-check
|
||||
test-count-ratchet-check: ## Check no workspace crate fell below its test floor or rose above its #[ignore] ceiling in the committed baseline
|
||||
@echo "🧪 Checking per-crate test-count ratchet..."
|
||||
|
||||
@@ -20,13 +20,13 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check gateway-stack-release-boundary-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check gateway-stack-release-boundary-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check gateway-stack-release-boundary-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
|
||||
@echo "✅ Fast development checks passed!"
|
||||
|
||||
@@ -61,7 +61,6 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_check_doc_paths.py
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
./scripts/check_s3s_footprint.sh --self-test
|
||||
./scripts/check_gateway_stack_release_boundary.sh --self-test
|
||||
./scripts/test_check_test_count_ratchet.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
|
||||
|
||||
@@ -721,7 +721,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage|website_stack_interim)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^conditional_delete_objects_test::/)
|
||||
|
||||
@@ -13,5 +13,5 @@
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=65
|
||||
s3_error_lines=217
|
||||
s3_error_lines=219
|
||||
ecstore_files=0
|
||||
|
||||
@@ -25,7 +25,7 @@ rustfs-data-usage 58 0
|
||||
rustfs-ecstore 7858 2
|
||||
rustfs-extension-schema 23 0
|
||||
rustfs-filemeta 361 0
|
||||
rustfs-gateway-adapter 36 0
|
||||
rustfs-gateway-adapter 74 0
|
||||
rustfs-heal 1128 3
|
||||
rustfs-heal-contracts 8 0
|
||||
rustfs-iam 415 0
|
||||
|
||||
@@ -97,10 +97,6 @@ runs:
|
||||
shell: bash
|
||||
run: ./scripts/check_s3s_footprint.sh
|
||||
|
||||
- name: Check gateway-stack release boundary
|
||||
shell: bash
|
||||
run: ./scripts/check_gateway_stack_release_boundary.sh
|
||||
|
||||
- name: Check per-crate test-count ratchet
|
||||
shell: bash
|
||||
run: ./scripts/check_test_count_ratchet.sh
|
||||
|
||||
@@ -1021,6 +1021,26 @@
|
||||
"policy_reason": "Preserve operator-disabled state; enabling requires a separate policy decision.",
|
||||
"origin": "baseline"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/manual-main-packages.yml",
|
||||
"name": "Manual main packages",
|
||||
"roles": [
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "event",
|
||||
"timezone": "UTC",
|
||||
"schedule": [],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "none",
|
||||
"result_owner": "manual-main-packages.yml",
|
||||
"disposition": "Packages the origin/main tip as workflow artifacts only. No GitHub Release or download-channel writes.",
|
||||
"monitor_reason": "Operator-requested packaging has no periodic freshness obligation.",
|
||||
"origin": "manual-main-packages"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/minio-interop.yml",
|
||||
"name": "minio-interop",
|
||||
|
||||
@@ -115,7 +115,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=4$' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -705,53 +705,6 @@ jobs:
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
test-and-lint-gateway-stack:
|
||||
name: Test and Lint (gateway-stack)
|
||||
# The gateway S3 stack (rustfs/backlog#2734) sits behind the rustfs
|
||||
# `gateway-stack` feature, which no other pull-request lane lints or tests.
|
||||
# Unlike the dormant feature lanes above, this one runs for every code
|
||||
# change: each operation slice puts its handler tests, its authorization
|
||||
# census and its end-to-end refusal test behind the feature, and a lane that
|
||||
# ran only on demand would let them rot behind a green pull request.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# Self-hosted for build scale and cache ownership: a workspace-scale Cargo
|
||||
# build that reads the ci-dev cache written on this image.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
# A feature that does not compile fails here; nothing below is skipped
|
||||
# into a pass.
|
||||
- name: Run clippy with gateway-stack
|
||||
run: cargo clippy --locked -p rustfs --all-targets --features gateway-stack -- -D warnings
|
||||
|
||||
# The gateway stack's own tests: the assembly, the authorizer table and
|
||||
# its census, the handler template, the migrated use cases, and the
|
||||
# embedded server driven through the whole middleware stack.
|
||||
- name: Run tests with gateway-stack
|
||||
env:
|
||||
# The same bounded link concurrency as the other feature lanes.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
cargo nextest run --profile ci --locked -p rustfs --features gateway-stack --no-tests fail \
|
||||
--status-level all --final-status-level all \
|
||||
-E 'test(/gateway_stack|gateway_handlers|server::stack::|list_buckets|legacy_stack_is_always_selectable/) | binary(embedded_gateway_stack_test)'
|
||||
|
||||
build-rustfs-debug-binary:
|
||||
name: Build RustFS Debug Binary
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
@@ -1626,7 +1579,6 @@ jobs:
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- uring-integration
|
||||
- e2e-tests
|
||||
@@ -1661,7 +1613,6 @@ jobs:
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- build-rustfs-debug-binary-rio-v2
|
||||
- uring-integration
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Manual packaging of the current origin/main tip. Workflow artifacts only.
|
||||
# Dispatch uses this file from the selected branch. The build fetches
|
||||
# refs/heads/main and checks out that commit, not the dispatch ref or a tag.
|
||||
|
||||
name: Manual main packages
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
architecture:
|
||||
description: "Linux targets: x86 (x86_64 musl+gnu), arm (aarch64 musl+gnu), or all"
|
||||
required: true
|
||||
type: choice
|
||||
options:
|
||||
- x86
|
||||
- arm
|
||||
- all
|
||||
default: all
|
||||
version:
|
||||
description: "Binary and package version: MAJOR.MINOR.PATCH, or MAJOR.MINOR.PATCH-(alpha|beta|rc).N. No leading v."
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
CARGO_INCREMENTAL: 0
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
name: Resolve main tip and version
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
outputs:
|
||||
source_sha: ${{ steps.pin.outputs.source_sha }}
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
toolchain: ${{ steps.inputs.outputs.toolchain }}
|
||||
console: ${{ steps.inputs.outputs.console }}
|
||||
matrix: ${{ steps.matrix.outputs.matrix }}
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: main
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Pin the tip of origin/main
|
||||
id: pin
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
|
||||
sha=$(git rev-parse FETCH_HEAD)
|
||||
git checkout --detach "$sha"
|
||||
test "$(git rev-parse HEAD)" = "$sha"
|
||||
printf 'source_sha=%s\n' "$sha" >> "$GITHUB_OUTPUT"
|
||||
printf 'Pinned origin/main %s\n' "$sha"
|
||||
|
||||
- name: Validate package version
|
||||
id: version
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
bash scripts/release/package_manual_artifacts.sh validate \
|
||||
--version "$PACKAGE_VERSION" \
|
||||
--arch x86_64 \
|
||||
--libc gnu | tee "$RUNNER_TEMP/package-version.txt"
|
||||
printf 'version=%s\n' "$PACKAGE_VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Install stable Rust
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
with:
|
||||
toolchain: stable
|
||||
|
||||
- name: Resolve console assets and toolchain
|
||||
id: inputs
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
toolchain=$(rustc --version | awk '{print $2}')
|
||||
[[ "$toolchain" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
|
||||
gh api repos/rustfs/console/releases/latest > console-release.json
|
||||
TOOLCHAIN="$toolchain" python3 - <<'PY'
|
||||
import json, os, sys
|
||||
sys.path.insert(0, "scripts/release")
|
||||
import candidate
|
||||
console = candidate.console_asset(candidate.read_json("console-release.json"))
|
||||
candidate.emit_output("console", json.dumps(console, separators=(",", ":")))
|
||||
candidate.emit_output("toolchain", os.environ["TOOLCHAIN"])
|
||||
print(f"toolchain={os.environ['TOOLCHAIN']}")
|
||||
print(f"console={console['tag']} {console['name']}")
|
||||
PY
|
||||
|
||||
- name: Select Linux targets
|
||||
id: matrix
|
||||
shell: bash
|
||||
env:
|
||||
ARCHITECTURE: ${{ inputs.architecture }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
matrix=$(bash scripts/release/package_manual_artifacts.sh matrix "$ARCHITECTURE")
|
||||
count=$(python3 -c 'import json,sys; print(len(json.loads(sys.argv[1])["include"]))' "$matrix")
|
||||
[[ "$count" -gt 0 ]]
|
||||
printf 'matrix=%s\n' "$matrix" >> "$GITHUB_OUTPUT"
|
||||
printf 'Selected %s target(s) for %s\n' "$count" "$ARCHITECTURE"
|
||||
|
||||
package:
|
||||
name: Package RustFS (${{ matrix.target_id }})
|
||||
needs: prepare
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
# x86_64 uses sm-standard-4, the official release compile pool: the workspace
|
||||
# release build needs that disk (placement rule: disk) and reads build.yml's
|
||||
# cache without writing it. aarch64 uses ubicloud-standard-16-arm so the
|
||||
# packaged binary and console smoke execute natively (placement rule: hardware).
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 210
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTUP_TOOLCHAIN: ${{ needs.prepare.outputs.toolchain }}
|
||||
RUSTFLAGS: "${{ matrix.rustflags }}"
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
|
||||
steps:
|
||||
- name: Checkout the pinned origin/main commit
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: ${{ needs.prepare.outputs.source_sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Confirm the checkout is that main commit
|
||||
shell: bash
|
||||
env:
|
||||
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$(git rev-parse HEAD)" = "$SOURCE_SHA"
|
||||
GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
|
||||
git merge-base --is-ancestor "$SOURCE_SHA" FETCH_HEAD
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: ${{ needs.prepare.outputs.toolchain }}
|
||||
target: ${{ matrix.target }}
|
||||
cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}${{ matrix.os == 'ubicloud-standard-16-arm' && '-ubicloud-arm' || '' }}
|
||||
# build.yml owns this key. This job only restores it.
|
||||
cache-save-if: 'false'
|
||||
install-cross-tools: ${{ matrix.cross }}
|
||||
install-build-packaging-tools: 'true'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Download static console assets
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CONSOLE_IDENTITY: ${{ needs.prepare.outputs.console }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PYCONSOLE'
|
||||
import json, os, pathlib, subprocess, zipfile
|
||||
value = json.loads(os.environ["CONSOLE_IDENTITY"])
|
||||
subprocess.run(["python3", ".github/actions/setup/download_release_asset.py",
|
||||
"--repository", "rustfs/console", "--tag", value["tag"], "--asset", value["name"],
|
||||
"--sha256", value["sha256"], "--output", "console.zip"], check=True)
|
||||
destination = pathlib.Path("rustfs/static")
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile("console.zip") as archive:
|
||||
for member in archive.infolist():
|
||||
path = pathlib.PurePosixPath(member.filename)
|
||||
if path.is_absolute() or ".." in path.parts or "\\" in member.filename:
|
||||
raise SystemExit("unsafe Console archive member")
|
||||
archive.extractall(destination)
|
||||
if not (destination / "index.html").is_file():
|
||||
raise SystemExit("Console archive has no index.html")
|
||||
PYCONSOLE
|
||||
rm -f console.zip
|
||||
|
||||
- name: Build RustFS
|
||||
shell: bash
|
||||
env:
|
||||
RUSTFS_BUILD_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
BUILD_OS: ${{ matrix.build_os }}
|
||||
BUILD_FEATURES: ${{ matrix.features }}
|
||||
BUILD_GLIBC: ${{ matrix.glibc }}
|
||||
BUILD_CROSS: ${{ matrix.cross }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
touch rustfs/build.rs
|
||||
|
||||
feature_args=()
|
||||
if [[ -n "$BUILD_FEATURES" ]]; then
|
||||
feature_args=(--features "$BUILD_FEATURES")
|
||||
fi
|
||||
|
||||
# gnu targets pin glibc 2.31 through cargo-zigbuild (<target>.<glibc>).
|
||||
target_arg="$BUILD_TARGET"
|
||||
if [[ -n "$BUILD_GLIBC" ]]; then
|
||||
target_arg="${target_arg}.${BUILD_GLIBC}"
|
||||
fi
|
||||
|
||||
profile_args=()
|
||||
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
|
||||
profile_args=(--config 'profile.release.strip="none"')
|
||||
fi
|
||||
|
||||
case "$BUILD_CROSS" in
|
||||
true)
|
||||
cargo zigbuild --release --locked --target "$target_arg" "${feature_args[@]}" "${profile_args[@]}" -p rustfs --bin rustfs
|
||||
;;
|
||||
false)
|
||||
cargo build --release --locked --target "$target_arg" "${feature_args[@]}" "${profile_args[@]}" -p rustfs --bin rustfs
|
||||
;;
|
||||
*)
|
||||
printf 'unexpected cross flag: %q\n' "$BUILD_CROSS" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
bin="target/${BUILD_TARGET}/release/rustfs"
|
||||
version_output=$("$bin" --version)
|
||||
first=${version_output%%$'\n'*}
|
||||
if [[ "$first" != "rustfs ${RUSTFS_BUILD_VERSION}" ]]; then
|
||||
printf 'binary version line is %q, expected %q\n' "$first" "rustfs ${RUSTFS_BUILD_VERSION}" >&2
|
||||
printf '%s\n' "$version_output" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
|
||||
! grep -Fxq "build os : ${BUILD_OS}" <<< "$version_output" || \
|
||||
! grep -Fxq "rust channel : stable-${BUILD_TARGET}" <<< "$version_output"; then
|
||||
printf 'release binary reports an unexpected compile target\n' >&2
|
||||
printf '%s\n' "$version_output" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$BUILD_GLIBC" ]]; then
|
||||
python3 scripts/release/check_glibc.py "$BUILD_GLIBC" "$bin"
|
||||
fi
|
||||
|
||||
- name: Generate the CPU symbol catalog
|
||||
if: endsWith(matrix.target, '-linux-gnu') && matrix.features == 'pyroscope'
|
||||
shell: bash
|
||||
env:
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo metadata --locked --format-version 1 --filter-platform "$BUILD_TARGET" \
|
||||
--features rustfs/pyroscope > "$RUNNER_TEMP/cpu-catalog-metadata.json"
|
||||
features=$(jq -ce '
|
||||
(.packages[] | select(.name == "rustfs") | .id) as $id |
|
||||
.resolve.nodes[] | select(.id == $id) | .features | map(select(. != "default")) | sort
|
||||
' "$RUNNER_TEMP/cpu-catalog-metadata.json")
|
||||
release_dir="target/${BUILD_TARGET}/release"
|
||||
rm -f "$release_dir/rustfs.cpu-symbol-catalog.json" "$release_dir/rustfs.cpu-symbol-catalog.sha256"
|
||||
cargo run --locked -p rustfs-connect-symbol-catalog -- \
|
||||
"$release_dir/rustfs" "$(git rev-parse HEAD)" "$BUILD_TARGET" "$features" \
|
||||
"$release_dir/rustfs.cpu-symbol-catalog.json" > "$release_dir/rustfs.cpu-symbol-catalog.sha256"
|
||||
|
||||
- name: Install DEB and RPM tools
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ruby ruby-dev build-essential rpm fakeroot cpio
|
||||
sudo gem install fpm --version 1.17.0 --no-document
|
||||
|
||||
- name: Package zip, deb, and rpm
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
ARCH: ${{ matrix.arch }}
|
||||
LIBC: ${{ matrix.libc }}
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
BUILD_FEATURES: ${{ matrix.features }}
|
||||
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
binary="target/${BUILD_TARGET}/release/rustfs"
|
||||
extra=()
|
||||
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
|
||||
extra=(
|
||||
--zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.json"
|
||||
--zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.sha256"
|
||||
)
|
||||
fi
|
||||
bash scripts/release/package_manual_artifacts.sh package \
|
||||
--version "$PACKAGE_VERSION" \
|
||||
--arch "$ARCH" \
|
||||
--libc "$LIBC" \
|
||||
--binary "$binary" \
|
||||
--output dist \
|
||||
"${extra[@]}"
|
||||
printf 'Packaged %s from origin/main %s\n' "$PACKAGE_VERSION" "$SOURCE_SHA"
|
||||
|
||||
- name: Verify the packaged binary
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
ARCH: ${{ matrix.arch }}
|
||||
LIBC: ${{ matrix.libc }}
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
BUILD_OS: ${{ matrix.build_os }}
|
||||
BUILD_FEATURES: ${{ matrix.features }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
stem="rustfs-linux-${ARCH}-${LIBC}-v${PACKAGE_VERSION}"
|
||||
package_file="dist/${stem}.zip"
|
||||
[[ -f "$package_file" && -f "dist/${stem}.deb" && -f "dist/${stem}.rpm" ]]
|
||||
[[ "$(find dist -maxdepth 1 -type f | wc -l)" -eq 3 ]]
|
||||
archive_members=$(unzip -Z1 "$package_file" | sort)
|
||||
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
|
||||
expected_members=$(printf '%s\n' rustfs rustfs.cpu-symbol-catalog.json rustfs.cpu-symbol-catalog.sha256 | sort)
|
||||
else
|
||||
expected_members=$(printf '%s\n' rustfs)
|
||||
fi
|
||||
if [[ "$archive_members" != "$expected_members" ]]; then
|
||||
printf 'Linux package does not contain the expected files\n' >&2
|
||||
printf '%s\n' "$archive_members" >&2
|
||||
exit 1
|
||||
fi
|
||||
binary_dir=$(mktemp -d)
|
||||
trap 'rm -rf "$binary_dir"' EXIT
|
||||
unzip -q "$package_file" rustfs -d "$binary_dir"
|
||||
version_output=$("$binary_dir/rustfs" --version)
|
||||
first=${version_output%%$'\n'*}
|
||||
if [[ "$first" != "rustfs ${PACKAGE_VERSION}" ]]; then
|
||||
printf 'packaged binary version line is %q, expected %q\n' "$first" "rustfs ${PACKAGE_VERSION}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
|
||||
! grep -Fxq "build os : ${BUILD_OS}" <<< "$version_output"; then
|
||||
printf 'packaged binary reports an unexpected compile target\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Verify packaged console
|
||||
if: matrix.target == 'x86_64-unknown-linux-gnu' || matrix.target == 'aarch64-unknown-linux-gnu'
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
ARCH: ${{ matrix.arch }}
|
||||
LIBC: ${{ matrix.libc }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
package_file="dist/rustfs-linux-${ARCH}-${LIBC}-v${PACKAGE_VERSION}.zip"
|
||||
|
||||
find_free_port() {
|
||||
python3 - <<'PY'
|
||||
import socket
|
||||
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
print(sock.getsockname()[1])
|
||||
PY
|
||||
}
|
||||
|
||||
package_dir=$(mktemp -d)
|
||||
data_dir=$(mktemp -d)
|
||||
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
|
||||
server_pid=""
|
||||
trap '
|
||||
if [[ -n "${server_pid:-}" ]]; then
|
||||
kill "$server_pid" 2>/dev/null || true
|
||||
wait "$server_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf "$package_dir" "$data_dir"
|
||||
' EXIT
|
||||
|
||||
unzip -q "$package_file" -d "$package_dir"
|
||||
api_port=$(find_free_port)
|
||||
console_port=$(find_free_port)
|
||||
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
|
||||
|
||||
"$package_dir/rustfs" server \
|
||||
--address "127.0.0.1:${api_port}" \
|
||||
--console-enable \
|
||||
--console-address "127.0.0.1:${console_port}" \
|
||||
--access-key console-smoke \
|
||||
--secret-key console-smoke-secret \
|
||||
"$data_dir" >"$log_file" 2>&1 &
|
||||
server_pid=$!
|
||||
|
||||
for _ in {1..40}; do
|
||||
response=$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)
|
||||
if [[ "$response" == 200\ text/html* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
|
||||
printf 'Console endpoint did not return 200 text/html: %s\n' "$response" >&2
|
||||
cat "$log_file"
|
||||
exit 1
|
||||
|
||||
- name: Upload packages
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: packages-${{ matrix.target_id }}-v${{ needs.prepare.outputs.version }}
|
||||
path: dist/*
|
||||
if-no-files-found: error
|
||||
compression-level: 0
|
||||
retention-days: 30
|
||||
@@ -112,9 +112,11 @@ jobs:
|
||||
# GitHub-hosted: reliably provides a running Docker daemon + buildx +
|
||||
# python3. See the header note (ci-2) for why the self-hosted sm-standard-4
|
||||
# label was abandoned (no docker.sock on the pod its only run landed on).
|
||||
# sm-standard-2 has no docker.sock either (weekly runs failed at "Enable
|
||||
# buildx" from 2026-09-27 to 2026-11-08), so keep this job on ubuntu-latest.
|
||||
# TODO(ci-8): scheduled-failure alerting (auto-open issue on a red weekly
|
||||
# run) is added by the ci-8 composite action; do not implement it here.
|
||||
runs-on: sm-standard-2
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
|
||||
Generated
+2
-2
@@ -9639,8 +9639,6 @@ dependencies = [
|
||||
"rustfs-ecstore",
|
||||
"rustfs-extension-schema",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-gateway",
|
||||
"rustfs-gateway-adapter",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-heal",
|
||||
"rustfs-heal-contracts",
|
||||
@@ -10065,8 +10063,10 @@ dependencies = [
|
||||
name = "rustfs-gateway-adapter"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"http 1.5.0",
|
||||
"rustfs-credentials",
|
||||
"rustfs-gateway",
|
||||
"rustfs-gateway-http",
|
||||
"rustfs-iam",
|
||||
"rustfs-policy",
|
||||
"serde_json",
|
||||
|
||||
@@ -155,6 +155,8 @@ rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.57.0", default-features = false }
|
||||
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
|
||||
+22
-4
@@ -24,9 +24,10 @@
|
||||
#
|
||||
# Supports cross-compilation for amd64 and arm64 via TARGETPLATFORM.
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
# Do not declare TARGETPLATFORM, BUILDPLATFORM or TARGETARCH here: a global
|
||||
# declaration without a value shadows BuildKit's automatic values, so the
|
||||
# stage-level re-declarations below see empty strings. They are only
|
||||
# declared inside the stage that uses them.
|
||||
ARG RUSTFS_BUILD_FEATURES=""
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
@@ -169,7 +170,24 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
;; \
|
||||
esac
|
||||
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
# TARGETARCH is only provided by BuildKit. Fall back to TARGETPLATFORM (for
|
||||
# example set by docker-compose) and then to the build host, like the cargo
|
||||
# step above, so every path that builds this file passes a real architecture.
|
||||
RUN set -eu; \
|
||||
rc_arch="${TARGETARCH:-}"; \
|
||||
if [ -z "${rc_arch}" ]; then \
|
||||
case "${TARGETPLATFORM:-}" in \
|
||||
linux/amd64) rc_arch=amd64 ;; \
|
||||
linux/arm64) rc_arch=arm64 ;; \
|
||||
*) \
|
||||
case "$(uname -m)" in \
|
||||
x86_64) rc_arch=amd64 ;; \
|
||||
aarch64|arm64) rc_arch=arm64 ;; \
|
||||
*) echo "TARGETARCH is empty and TARGETPLATFORM='${TARGETPLATFORM:-}' is not linux/amd64 or linux/arm64; cannot choose an rc architecture on $(uname -m)" >&2; exit 1 ;; \
|
||||
esac ;; \
|
||||
esac; \
|
||||
fi; \
|
||||
sh /usr/src/rustfs/scripts/download_rc.sh "${rc_arch}" musl "${RC_RELEASE}" /usr/local/bin/rc
|
||||
|
||||
# -----------------------------
|
||||
# Development stage (keeps toolchain)
|
||||
|
||||
@@ -29,6 +29,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
| **dual-stack differential** | `dual_stack_differential` | Two single-node servers answer one SDK script for all 73 S3 operations and every differing `(op, field, a, b)` is reported. Side A is the binary under test on `RUSTFS_S3_STACK=legacy`; side B runs `RUSTFS_E2E_STACK_B`. Set `RUSTFS_E2E_REFERENCE_BINARY` to a released `rustfs` binary to make side A that binary instead (started without `RUSTFS_S3_STACK`, which older releases do not know). A missing, non-file or non-executable path fails the run; it never skips or falls back to the same-binary mode. The report in `<target>/dual-stack/<run>.json` records `reference_side` and each side's `--version` output |
|
||||
|
||||
The external-tool `storage_metric_ownership_test` validates the OTLP/Collector/Prometheus path, including a rolling upgrade and node failures. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for its required binaries and focused command.
|
||||
|
||||
|
||||
@@ -774,6 +774,18 @@ impl RustFSTestEnvironment {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Like [`Self::start_rustfs_server_from_binary`], but leaves other running RustFS processes
|
||||
/// alone, so that two servers of different binaries can be alive at once.
|
||||
pub async fn start_rustfs_server_from_binary_without_cleanup(
|
||||
&mut self,
|
||||
binary_path: &Path,
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start RustFS server with basic configuration
|
||||
pub async fn start_rustfs_server(&mut self, extra_args: Vec<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner(extra_args, &[], true).await
|
||||
|
||||
@@ -1285,7 +1285,9 @@ async fn list_parts_disk_loss_after_metadata_read(
|
||||
let error = result.expect_err("one missing directory and three unavailable participants cannot prove an empty upload");
|
||||
let response = error.raw_response().expect("ListParts must return an S3 service response");
|
||||
assert_eq!(response.status().as_u16(), 503);
|
||||
assert_eq!(error.as_service_error().and_then(|err| err.meta().code()), Some("SlowDownRead"));
|
||||
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
|
||||
// ListParts read-quorum failure is reported as ServiceUnavailable.
|
||||
assert_eq!(error.as_service_error().and_then(|err| err.meta().code()), Some("ServiceUnavailable"));
|
||||
assert!(response.headers().get("x-rustfs-error-cause").is_none());
|
||||
assert!(response.headers().get("retry-after").is_none());
|
||||
assert!(
|
||||
|
||||
@@ -493,7 +493,12 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
|
||||
assert!(legacy.parts().is_empty());
|
||||
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
|
||||
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
|
||||
assert_eq!(current.as_service_error().and_then(|error| error.meta().code()), Some("SlowDownRead"));
|
||||
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
|
||||
// ListParts read-quorum failure is reported as ServiceUnavailable.
|
||||
assert_eq!(
|
||||
current.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable")
|
||||
);
|
||||
dist.cluster.start_node_from_binary(3, previous).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
|
||||
for (path, before) in original_files {
|
||||
|
||||
@@ -232,6 +232,12 @@ pub(crate) struct Report {
|
||||
pub run: String,
|
||||
pub stack_a: String,
|
||||
pub stack_b: String,
|
||||
/// Which side is the pinned release binary: `"a"`, or `None` when both sides are the binary
|
||||
/// under test.
|
||||
pub reference_side: Option<&'static str>,
|
||||
/// `--version` output of the binary each side actually ran.
|
||||
pub version_a: String,
|
||||
pub version_b: String,
|
||||
pub region_a: Option<String>,
|
||||
pub region_b: Option<String>,
|
||||
pub compared_ops: usize,
|
||||
@@ -480,10 +486,13 @@ mod tests {
|
||||
fn the_report_lists_each_difference_as_op_field_a_b() {
|
||||
let directory = std::env::temp_dir().join(format!("dual-stack-report-{}", uuid::Uuid::new_v4()));
|
||||
let report = Report {
|
||||
schema: 1,
|
||||
schema: 2,
|
||||
run: "unit".into(),
|
||||
stack_a: "legacy".into(),
|
||||
stack_b: "legacy".into(),
|
||||
reference_side: Some("a"),
|
||||
version_a: "rustfs 1.0.4".into(),
|
||||
version_b: "rustfs 1.1.0".into(),
|
||||
region_a: None,
|
||||
region_b: Some("eu-west-1".into()),
|
||||
compared_ops: 1,
|
||||
@@ -498,6 +507,9 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&directory);
|
||||
assert_eq!(path.file_name().and_then(|name| name.to_str()), Some("unit.json"));
|
||||
assert_eq!(json["compared_ops"], 1);
|
||||
assert_eq!(json["reference_side"], "a");
|
||||
assert_eq!(json["version_a"], "rustfs 1.0.4");
|
||||
assert_eq!(json["version_b"], "rustfs 1.1.0");
|
||||
assert_eq!(
|
||||
json["differences"],
|
||||
serde_json::json!([{
|
||||
|
||||
@@ -14,7 +14,10 @@
|
||||
|
||||
//! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
|
||||
//!
|
||||
//! Stack A always runs `RUSTFS_S3_STACK=legacy`. Stack B runs the value of `RUSTFS_E2E_STACK_B`
|
||||
//! Stack A runs `RUSTFS_S3_STACK=legacy` of the binary under test, unless `RUSTFS_E2E_REFERENCE_BINARY`
|
||||
//! names a released binary: then A is that binary, started without `RUSTFS_S3_STACK` (older
|
||||
//! releases do not know it), and a bad path fails the run instead of falling back (see
|
||||
//! `reference`). The report names which side is the reference and the `--version` each side ran. Stack B runs the value of `RUSTFS_E2E_STACK_B`
|
||||
//! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
|
||||
//! set, starts B in another region so that a difference can be injected by hand. Both servers
|
||||
//! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
|
||||
@@ -23,16 +26,20 @@
|
||||
//! is process-global, so two stacks in one process would read each other's buckets.
|
||||
|
||||
mod compare;
|
||||
mod reference;
|
||||
mod scenarios;
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, workspace_root};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path, workspace_root};
|
||||
use compare::{Difference, Observation, Report, compare, compared_cases, compared_ops, render};
|
||||
use std::path::PathBuf;
|
||||
use reference::{SideA, binary_version};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const STACK_A: &str = "legacy";
|
||||
/// Label of side A in the report when it is the released reference binary.
|
||||
const STACK_A_REFERENCE: &str = "reference";
|
||||
const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
|
||||
const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
|
||||
/// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
|
||||
@@ -52,22 +59,31 @@ const CONTROL_REGION_B: &str = "eu-west-1";
|
||||
const MIN_NORMAL_SUCCESSES: usize = 68;
|
||||
|
||||
struct PairConfig {
|
||||
side_a: SideA,
|
||||
stack_b: String,
|
||||
region_a: Option<String>,
|
||||
region_b: Option<String>,
|
||||
}
|
||||
|
||||
impl PairConfig {
|
||||
fn from_env() -> Self {
|
||||
Self {
|
||||
fn from_env() -> Result<Self, BoxError> {
|
||||
Ok(Self {
|
||||
side_a: SideA::from_env()?,
|
||||
stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
|
||||
region_a: None,
|
||||
region_b: std::env::var(ENV_STACK_B_REGION).ok(),
|
||||
})
|
||||
}
|
||||
|
||||
fn stack_a_label(&self) -> &'static str {
|
||||
match self.side_a {
|
||||
SideA::SameBinary => STACK_A,
|
||||
SideA::Reference(_) => STACK_A_REFERENCE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts one server on `stack`, overriding its region only when asked.
|
||||
/// Starts one server of the binary under test on `stack`, overriding its region only when asked.
|
||||
async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = vec![("RUSTFS_S3_STACK", stack)];
|
||||
@@ -78,12 +94,28 @@ async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvi
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// Starts one server of a released binary. `RUSTFS_S3_STACK` is not passed: older releases do not
|
||||
/// know it and have exactly one stack.
|
||||
async fn start_reference(binary: &Path, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = Vec::new();
|
||||
if let Some(region) = region {
|
||||
vars.push(("RUSTFS_REGION", region));
|
||||
}
|
||||
env.start_rustfs_server_from_binary_without_cleanup(binary, vec![], &vars)
|
||||
.await?;
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// Starts stack A and stack B side by side, each with its own port and data directory.
|
||||
async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
|
||||
tokio::try_join!(
|
||||
start_stack(STACK_A, config.region_a.as_deref()),
|
||||
start_stack(&config.stack_b, config.region_b.as_deref())
|
||||
)
|
||||
let side_a = async {
|
||||
match &config.side_a {
|
||||
SideA::SameBinary => start_stack(STACK_A, config.region_a.as_deref()).await,
|
||||
SideA::Reference(binary) => start_reference(binary, config.region_a.as_deref()).await,
|
||||
}
|
||||
};
|
||||
tokio::try_join!(side_a, start_stack(&config.stack_b, config.region_b.as_deref()))
|
||||
}
|
||||
|
||||
fn report_directory() -> PathBuf {
|
||||
@@ -104,16 +136,34 @@ struct Run {
|
||||
|
||||
/// Runs the script against both stacks concurrently and writes the report.
|
||||
async fn run_differential(label: &str, config: &PairConfig) -> Run {
|
||||
let binary_under_test = rustfs_binary_path();
|
||||
let (version_a, version_b) = match &config.side_a {
|
||||
SideA::SameBinary => {
|
||||
let version = binary_version(&binary_under_test)
|
||||
.await
|
||||
.expect("capture the binary under test version");
|
||||
(version.clone(), version)
|
||||
}
|
||||
SideA::Reference(binary) => (
|
||||
binary_version(binary).await.expect("capture the reference binary version"),
|
||||
binary_version(&binary_under_test)
|
||||
.await
|
||||
.expect("capture the binary under test version"),
|
||||
),
|
||||
};
|
||||
let (server_a, server_b) = spawn_pair(config).await.expect("start stack A and stack B");
|
||||
let (a, b) = tokio::join!(Box::pin(scenarios::run(&server_a)), Box::pin(scenarios::run(&server_b)));
|
||||
let millis = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_or(0, |elapsed| elapsed.as_millis());
|
||||
let report = Report {
|
||||
schema: 1,
|
||||
schema: 2,
|
||||
run: format!("{label}-{millis}-{}", std::process::id()),
|
||||
stack_a: STACK_A.to_owned(),
|
||||
stack_a: config.stack_a_label().to_owned(),
|
||||
stack_b: config.stack_b.clone(),
|
||||
reference_side: matches!(config.side_a, SideA::Reference(_)).then_some("a"),
|
||||
version_a,
|
||||
version_b,
|
||||
region_a: config.region_a.clone(),
|
||||
region_b: config.region_b.clone(),
|
||||
compared_ops: compared_ops(&a, &b).len(),
|
||||
@@ -131,7 +181,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn both_stacks_answer_every_operation_identically() {
|
||||
init_logging();
|
||||
let config = PairConfig::from_env();
|
||||
let config = PairConfig::from_env().expect("RUSTFS_E2E_REFERENCE_BINARY must be valid when set");
|
||||
let run = run_differential("self-comparison", &config).await;
|
||||
let report = &run.report;
|
||||
|
||||
@@ -146,9 +196,11 @@ mod tests {
|
||||
assert_eq!(report.compared_ops, scenarios::OPERATIONS.len(), "report: {}", run.path.display());
|
||||
assert!(
|
||||
report.differences.is_empty(),
|
||||
"stack {} and stack {} differ on {} of {} cases (report: {}):\n{}",
|
||||
"stack {} ({}) and stack {} ({}) differ on {} of {} cases (report: {}):\n{}",
|
||||
report.stack_a,
|
||||
report.version_a,
|
||||
report.stack_b,
|
||||
report.version_b,
|
||||
report.differences.len(),
|
||||
report.compared_cases,
|
||||
run.path.display(),
|
||||
@@ -160,7 +212,11 @@ mod tests {
|
||||
.iter()
|
||||
.filter(|observation| observation.case == "negative" && !(400..500).contains(&observation.status))
|
||||
.collect();
|
||||
assert!(not_4xx.is_empty(), "negative cases must fail with a 4xx on {STACK_A}: {not_4xx:#?}");
|
||||
assert!(
|
||||
not_4xx.is_empty(),
|
||||
"negative cases must fail with a 4xx on side A ({}): {not_4xx:#?}",
|
||||
report.version_a
|
||||
);
|
||||
let successes = run
|
||||
.a
|
||||
.iter()
|
||||
@@ -168,7 +224,8 @@ mod tests {
|
||||
.count();
|
||||
assert!(
|
||||
successes >= MIN_NORMAL_SUCCESSES,
|
||||
"only {successes} normal cases succeeded on {STACK_A}: {:#?}",
|
||||
"only {successes} normal cases succeeded on side A ({}): {:#?}",
|
||||
report.version_a,
|
||||
run.a
|
||||
);
|
||||
}
|
||||
@@ -177,6 +234,7 @@ mod tests {
|
||||
async fn differential_detects_an_injected_difference() {
|
||||
init_logging();
|
||||
let config = PairConfig {
|
||||
side_a: SideA::SameBinary,
|
||||
stack_b: STACK_A.to_owned(),
|
||||
region_a: Some(CONTROL_REGION_A.to_owned()),
|
||||
region_b: Some(CONTROL_REGION_B.to_owned()),
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Which binary side A of the differential runs, and what it reports about itself.
|
||||
//!
|
||||
//! `RUSTFS_E2E_REFERENCE_BINARY` names a released `rustfs` binary to use as the reference. Unset,
|
||||
//! side A is the binary under test on `RUSTFS_S3_STACK=legacy`. Set, it must resolve to an
|
||||
//! executable file or the run fails: a reference that was asked for and silently replaced by the
|
||||
//! binary under test would report an agreement with itself as agreement with the release.
|
||||
|
||||
use std::ffi::OsString;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
pub(super) const ENV_REFERENCE_BINARY: &str = "RUSTFS_E2E_REFERENCE_BINARY";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(super) enum SideA {
|
||||
/// Both sides are the binary under test; A runs `RUSTFS_S3_STACK=legacy`.
|
||||
SameBinary,
|
||||
/// A is this released binary, started without `RUSTFS_S3_STACK`.
|
||||
Reference(PathBuf),
|
||||
}
|
||||
|
||||
impl SideA {
|
||||
pub(super) fn from_env() -> Result<Self, BoxError> {
|
||||
Self::resolve(std::env::var_os(ENV_REFERENCE_BINARY))
|
||||
}
|
||||
|
||||
/// `None` is the only value that selects same-binary mode; every other value must validate.
|
||||
pub(super) fn resolve(value: Option<OsString>) -> Result<Self, BoxError> {
|
||||
let Some(value) = value else {
|
||||
return Ok(Self::SameBinary);
|
||||
};
|
||||
let path = PathBuf::from(value);
|
||||
validate_executable(&path)?;
|
||||
Ok(Self::Reference(path))
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_executable(path: &Path) -> Result<(), BoxError> {
|
||||
if path.as_os_str().is_empty() {
|
||||
return Err(format!("{ENV_REFERENCE_BINARY} is set but empty").into());
|
||||
}
|
||||
let metadata = std::fs::metadata(path)
|
||||
.map_err(|error| format!("{ENV_REFERENCE_BINARY} points to {}, which cannot be read: {error}", path.display()))?;
|
||||
if !metadata.is_file() {
|
||||
return Err(format!("{ENV_REFERENCE_BINARY} points to {}, which is not a file", path.display()).into());
|
||||
}
|
||||
if !is_executable(&metadata) {
|
||||
return Err(format!("{ENV_REFERENCE_BINARY} points to {}, which is not executable", path.display()).into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn is_executable(metadata: &std::fs::Metadata) -> bool {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
metadata.permissions().mode() & 0o111 != 0
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn is_executable(_metadata: &std::fs::Metadata) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// What `<binary> --version` printed, trimmed. Fails when the binary does not run, exits
|
||||
/// non-zero, or prints nothing, because an empty version would report nothing as observed.
|
||||
pub(super) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
|
||||
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
|
||||
if !output.status.success() {
|
||||
return Err(format!("{} --version failed with {}", binary.display(), output.status).into());
|
||||
}
|
||||
let version = String::from_utf8_lossy(&output.stdout).trim().to_owned();
|
||||
if version.is_empty() {
|
||||
return Err(format!("{} --version printed nothing", binary.display()).into());
|
||||
}
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
fn script(dir: &Path, name: &str, body: &str, mode: u32) -> PathBuf {
|
||||
let path = dir.join(name);
|
||||
std::fs::write(&path, body).expect("write the stand-in");
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(mode)).expect("chmod");
|
||||
path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_path_is_an_error_not_a_fallback() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let error = SideA::resolve(Some(dir.path().join("absent").into_os_string())).expect_err("must fail closed");
|
||||
assert!(error.to_string().contains("cannot be read"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_executable_file_is_an_error() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = script(dir.path(), "rustfs", "#!/bin/sh\necho v\n", 0o644);
|
||||
let error = SideA::resolve(Some(path.into_os_string())).expect_err("must fail closed");
|
||||
assert!(error.to_string().contains("not executable"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_is_an_error() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let error = SideA::resolve(Some(dir.path().as_os_str().to_owned())).expect_err("must fail closed");
|
||||
assert!(error.to_string().contains("not a file"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_value_is_an_error() {
|
||||
let error = SideA::resolve(Some(OsString::new())).expect_err("must fail closed");
|
||||
assert!(error.to_string().contains("empty"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unset_selects_same_binary_mode() {
|
||||
assert_eq!(SideA::resolve(None).expect("unset is valid"), SideA::SameBinary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn executable_file_selects_reference_mode() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = script(dir.path(), "rustfs", "#!/bin/sh\necho v\n", 0o755);
|
||||
assert_eq!(
|
||||
SideA::resolve(Some(path.clone().into_os_string())).expect("valid"),
|
||||
SideA::Reference(path)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn version_output_is_captured_from_the_binary() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = script(dir.path(), "rustfs", "#!/bin/sh\necho 'rustfs 1.0.4 (abc123)'\n", 0o755);
|
||||
assert_eq!(binary_version(&path).await.expect("version"), "rustfs 1.0.4 (abc123)");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failing_or_silent_version_is_an_error() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let failing = script(dir.path(), "failing", "#!/bin/sh\nexit 3\n", 0o755);
|
||||
assert!(binary_version(&failing).await.is_err());
|
||||
let silent = script(dir.path(), "silent", "#!/bin/sh\nexit 0\n", 0o755);
|
||||
assert!(binary_version(&silent).await.is_err());
|
||||
}
|
||||
}
|
||||
@@ -301,11 +301,6 @@ mod bucket_logging_test;
|
||||
#[cfg(test)]
|
||||
mod website_hosting_test;
|
||||
|
||||
// Website hosting and the metadata listings on the gateway stack, compared with the legacy
|
||||
// stack on one data directory (rustfs/backlog#2734 T2.11; R-064).
|
||||
#[cfg(test)]
|
||||
mod website_stack_interim_test;
|
||||
|
||||
// Multipart control API auth regression tests
|
||||
#[cfg(test)]
|
||||
mod multipart_auth_test;
|
||||
|
||||
@@ -1,475 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Website hosting and the `?metadata=true` listings on the gateway stack (rustfs/backlog#2734
|
||||
//! T2.11, PR "W"; release blocker R-064).
|
||||
//!
|
||||
//! Responsible for: proving that a server on `RUSTFS_S3_STACK=gateway` answers website-host
|
||||
//! requests and the two metadata listings exactly as the legacy stack does. One data directory,
|
||||
//! one binary, two processes in turn: the legacy stack seeds the buckets (the gateway stack cannot
|
||||
//! write them yet) and answers a fixed set of requests, then the same directory is restarted on
|
||||
//! the gateway stack and answers the same requests. Each answer is compared on status, `Location`,
|
||||
//! `Content-Type`, `Vary`, the CORS headers and the body bytes. The legacy answers are also held
|
||||
//! to what a website must answer, so that two stacks failing the same way cannot pass, and one
|
||||
//! request the gateway stack must NOT hand to legacy is held to differ, so that a restart that
|
||||
//! silently kept the legacy stack cannot pass either.
|
||||
//! NOT responsible for: the comparison against the released 1.0.5 binary
|
||||
//! (`.config/reference-binary.toml`); this harness has no reference side yet, so what is proved
|
||||
//! here is gateway == legacy of the candidate, not candidate == 1.0.5.
|
||||
//! Upstream: the routes under test are `rustfs/src/server/stack/{website,metadata}.rs`.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration};
|
||||
use http::Method;
|
||||
use http::header::HOST;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::time::Duration;
|
||||
|
||||
type TestResult<T = ()> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const WEBSITE_DOMAIN: &str = "website.test";
|
||||
const PUBLIC_BUCKET: &str = "interim-site-public";
|
||||
const PRIVATE_BUCKET: &str = "interim-site-private";
|
||||
const NO_LIST_USER: &str = "interimsitenolist";
|
||||
const NO_LIST_SECRET: &str = "interim-site-no-list-secret";
|
||||
const ORIGIN: &str = "https://app.example";
|
||||
/// The response headers compared besides status, `Location` and body.
|
||||
const COMPARED_HEADERS: [&str; 8] = [
|
||||
"content-type",
|
||||
"vary",
|
||||
"access-control-allow-origin",
|
||||
"access-control-allow-methods",
|
||||
"access-control-allow-headers",
|
||||
"access-control-expose-headers",
|
||||
"access-control-allow-credentials",
|
||||
"access-control-max-age",
|
||||
];
|
||||
|
||||
/// What one request was answered with, reduced to the compared fields.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
struct Answer {
|
||||
status: u16,
|
||||
location: Option<String>,
|
||||
headers: Vec<(&'static str, String)>,
|
||||
body: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Answer {
|
||||
async fn of(response: reqwest::Response) -> TestResult<Self> {
|
||||
let status = response.status().as_u16();
|
||||
let location = header_text(response.headers(), "location");
|
||||
let headers = COMPARED_HEADERS
|
||||
.iter()
|
||||
.filter_map(|name| header_text(response.headers(), name).map(|value| (*name, value)))
|
||||
.collect();
|
||||
let body = response.bytes().await?.to_vec();
|
||||
Ok(Self {
|
||||
status,
|
||||
location,
|
||||
headers,
|
||||
body,
|
||||
})
|
||||
}
|
||||
|
||||
fn body_text(&self) -> String {
|
||||
String::from_utf8_lossy(&self.body).into_owned()
|
||||
}
|
||||
}
|
||||
|
||||
/// Every value of `name`, joined, so a second `Vary` line is a difference too.
|
||||
fn header_text(headers: &reqwest::header::HeaderMap, name: &str) -> Option<String> {
|
||||
let values: Vec<&str> = headers
|
||||
.get_all(name)
|
||||
.iter()
|
||||
.map(|value| value.to_str().unwrap_or("<not UTF-8>"))
|
||||
.collect();
|
||||
(!values.is_empty()).then(|| values.join(" | "))
|
||||
}
|
||||
|
||||
/// A local client that reports a redirect instead of following it: the 302 and its
|
||||
/// `Location` are what is compared.
|
||||
fn no_redirect_client() -> reqwest::Client {
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("local http client")
|
||||
}
|
||||
|
||||
fn website_host(bucket: &str) -> String {
|
||||
format!("{bucket}.{WEBSITE_DOMAIN}")
|
||||
}
|
||||
|
||||
/// An unsigned request to `target` on this server, addressed to `host`.
|
||||
async fn anonymous(
|
||||
env: &RustFSTestEnvironment,
|
||||
method: Method,
|
||||
host: &str,
|
||||
target: &str,
|
||||
extra: &[(&str, &str)],
|
||||
) -> TestResult<Answer> {
|
||||
let mut request = no_redirect_client()
|
||||
.request(method, format!("{}{target}", env.url))
|
||||
.header(HOST, host)
|
||||
.header("origin", ORIGIN);
|
||||
for (name, value) in extra {
|
||||
request = request.header(*name, *value);
|
||||
}
|
||||
Answer::of(request.send().await?).await
|
||||
}
|
||||
|
||||
/// A SigV4 request to `target` on this server, addressed and signed for `host`.
|
||||
async fn signed(
|
||||
env: &RustFSTestEnvironment,
|
||||
host: &str,
|
||||
target: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> TestResult<Answer> {
|
||||
let url = format!("{}{target}", env.url);
|
||||
let request = http::Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri(url.as_str())
|
||||
.header(HOST, host)
|
||||
.header("origin", ORIGIN)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
|
||||
.body(Body::empty())?;
|
||||
let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1");
|
||||
let mut request = no_redirect_client().get(url);
|
||||
for (name, value) in signed.headers() {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
Answer::of(request.send().await?).await
|
||||
}
|
||||
|
||||
/// The server's own address: a path-style request, not a website one.
|
||||
fn server_host(env: &RustFSTestEnvironment) -> &str {
|
||||
&env.address
|
||||
}
|
||||
|
||||
/// Writes everything the comparison reads, through the legacy stack.
|
||||
async fn seed(env: &RustFSTestEnvironment) -> TestResult {
|
||||
let s3 = env.create_s3_client();
|
||||
for bucket in [PUBLIC_BUCKET, PRIVATE_BUCKET] {
|
||||
s3.create_bucket().bucket(bucket).send().await?;
|
||||
for (key, body) in [
|
||||
("index.html", "home"),
|
||||
("404.html", "not here"),
|
||||
("section/index.html", "section page"),
|
||||
] {
|
||||
s3.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.content_type("text/html")
|
||||
.metadata("colour", "blue")
|
||||
.body(ByteStream::from(body.as_bytes().to_vec()))
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
s3.put_bucket_website()
|
||||
.bucket(bucket)
|
||||
.website_configuration(
|
||||
WebsiteConfiguration::builder()
|
||||
.index_document(IndexDocument::builder().suffix("index.html").build()?)
|
||||
.error_document(ErrorDocument::builder().key("404.html").build()?)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
// Only the public bucket is readable anonymously, and only by GetObject: no listing.
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": format!("arn:aws:s3:::{PUBLIC_BUCKET}/*")
|
||||
}]
|
||||
});
|
||||
s3.put_bucket_policy()
|
||||
.bucket(PUBLIC_BUCKET)
|
||||
.policy(policy.to_string())
|
||||
.send()
|
||||
.await?;
|
||||
s3.put_bucket_cors()
|
||||
.bucket(PUBLIC_BUCKET)
|
||||
.cors_configuration(
|
||||
CorsConfiguration::builder()
|
||||
.cors_rules(
|
||||
CorsRule::builder()
|
||||
.allowed_origins(ORIGIN)
|
||||
.allowed_methods("GET")
|
||||
.allowed_methods("HEAD")
|
||||
.expose_headers("ETag")
|
||||
.build()?,
|
||||
)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
// A real user whose policy grants nothing, so it may not list either bucket.
|
||||
admin_ok(
|
||||
env,
|
||||
Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={NO_LIST_USER}"),
|
||||
Some(serde_json::json!({ "secretKey": NO_LIST_SECRET, "status": "enabled" }).to_string()),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// IAM loads its users in the background after a start, so a user unknown for a moment is
|
||||
/// answered `InvalidAccessKeyId`; wait until the user without a policy is known again.
|
||||
async fn wait_until_the_user_is_loaded(env: &RustFSTestEnvironment) -> TestResult {
|
||||
let target = format!("/{PUBLIC_BUCKET}?list-type=2&metadata=true");
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
let answer = signed(env, server_host(env), &target, NO_LIST_USER, NO_LIST_SECRET).await?;
|
||||
if !answer.body_text().contains("InvalidAccessKeyId") {
|
||||
return Ok(());
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!("{NO_LIST_USER} is still unknown: {answer:?}").into());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// The fixed requests, in order, each with the name the comparison reports it under.
|
||||
async fn observe(env: &RustFSTestEnvironment) -> TestResult<Vec<(&'static str, Answer)>> {
|
||||
let public = website_host(PUBLIC_BUCKET);
|
||||
let private = website_host(PRIVATE_BUCKET);
|
||||
let mut answers = Vec::new();
|
||||
wait_until_the_user_is_loaded(env).await?;
|
||||
|
||||
let head = anonymous(env, Method::HEAD, &public, "/", &[]).await?;
|
||||
let etag = no_redirect_client()
|
||||
.head(format!("{}/", env.url))
|
||||
.header(HOST, &public)
|
||||
.send()
|
||||
.await?
|
||||
.headers()
|
||||
.get("etag")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned)
|
||||
.ok_or("the website index document has no ETag")?;
|
||||
answers.push(("website GET /", anonymous(env, Method::GET, &public, "/", &[]).await?));
|
||||
answers.push(("website HEAD /", head));
|
||||
answers.push(("website GET directory", anonymous(env, Method::GET, &public, "/section", &[]).await?));
|
||||
answers.push(("website GET directory/", anonymous(env, Method::GET, &public, "/section/", &[]).await?));
|
||||
answers.push((
|
||||
"website GET missing key",
|
||||
anonymous(env, Method::GET, &public, "/missing.html", &[]).await?,
|
||||
));
|
||||
answers.push((
|
||||
"website GET Range",
|
||||
anonymous(env, Method::GET, &public, "/index.html", &[("range", "bytes=0-1")]).await?,
|
||||
));
|
||||
answers.push((
|
||||
"website GET If-None-Match",
|
||||
anonymous(env, Method::GET, &public, "/", &[("if-none-match", etag.as_str())]).await?,
|
||||
));
|
||||
answers.push(("website POST", anonymous(env, Method::POST, &public, "/", &[]).await?));
|
||||
answers.push(("website GET bare domain", anonymous(env, Method::GET, WEBSITE_DOMAIN, "/", &[]).await?));
|
||||
answers.push(("website GET private bucket", anonymous(env, Method::GET, &private, "/", &[]).await?));
|
||||
answers.push((
|
||||
"website GET private bucket, owner-signed",
|
||||
signed(env, &private, "/", &env.access_key, &env.secret_key).await?,
|
||||
));
|
||||
answers.push((
|
||||
"website GET public bucket, owner-signed",
|
||||
signed(env, &public, "/", &env.access_key, &env.secret_key).await?,
|
||||
));
|
||||
let host = server_host(env);
|
||||
answers.push((
|
||||
"ListObjectsV2 metadata=true",
|
||||
signed(
|
||||
env,
|
||||
host,
|
||||
&format!("/{PUBLIC_BUCKET}?list-type=2&metadata=true"),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await?,
|
||||
));
|
||||
answers.push((
|
||||
"ListObjectVersions metadata=true",
|
||||
signed(
|
||||
env,
|
||||
host,
|
||||
&format!("/{PUBLIC_BUCKET}?versions&metadata=true"),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await?,
|
||||
));
|
||||
answers.push((
|
||||
"ListObjectsV2 metadata=true, no list permission",
|
||||
signed(
|
||||
env,
|
||||
host,
|
||||
&format!("/{PUBLIC_BUCKET}?list-type=2&metadata=true"),
|
||||
NO_LIST_USER,
|
||||
NO_LIST_SECRET,
|
||||
)
|
||||
.await?,
|
||||
));
|
||||
answers.push((
|
||||
"ListObjectsV2 metadata=true, anonymous",
|
||||
anonymous(env, Method::GET, host, &format!("/{PUBLIC_BUCKET}?list-type=2&metadata=true"), &[]).await?,
|
||||
));
|
||||
Ok(answers)
|
||||
}
|
||||
|
||||
/// A plain ListObjectsV2: not a metadata listing, so the gateway stack must answer it itself.
|
||||
async fn plain_listing(env: &RustFSTestEnvironment) -> TestResult<Answer> {
|
||||
signed(
|
||||
env,
|
||||
server_host(env),
|
||||
&format!("/{PUBLIC_BUCKET}?list-type=2"),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
fn answer<'a>(answers: &'a [(&'static str, Answer)], name: &str) -> &'a Answer {
|
||||
&answers
|
||||
.iter()
|
||||
.find(|(named, _)| *named == name)
|
||||
.unwrap_or_else(|| panic!("no answer named {name}"))
|
||||
.1
|
||||
}
|
||||
|
||||
/// What a website and the metadata listings must answer, whichever stack serves them.
|
||||
fn assert_is_a_website(answers: &[(&'static str, Answer)]) {
|
||||
let get = answer(answers, "website GET /");
|
||||
assert_eq!((get.status, get.body_text().as_str()), (200, "home"), "{get:?}");
|
||||
let head = answer(answers, "website HEAD /");
|
||||
assert_eq!((head.status, head.body.len()), (200, 0), "{head:?}");
|
||||
let directory = answer(answers, "website GET directory");
|
||||
assert_eq!(
|
||||
(directory.status, directory.location.as_deref()),
|
||||
(302, Some("/section/")),
|
||||
"{directory:?}"
|
||||
);
|
||||
let section = answer(answers, "website GET directory/");
|
||||
assert_eq!((section.status, section.body_text().as_str()), (200, "section page"), "{section:?}");
|
||||
let missing = answer(answers, "website GET missing key");
|
||||
assert_eq!((missing.status, missing.body_text().as_str()), (404, "not here"), "{missing:?}");
|
||||
let range = answer(answers, "website GET Range");
|
||||
assert_eq!((range.status, range.body_text().as_str()), (206, "ho"), "{range:?}");
|
||||
let not_modified = answer(answers, "website GET If-None-Match");
|
||||
assert_eq!((not_modified.status, not_modified.body.len()), (304, 0), "{not_modified:?}");
|
||||
assert_eq!(answer(answers, "website POST").status, 405);
|
||||
assert_eq!(answer(answers, "website GET bare domain").status, 404);
|
||||
for name in ["website GET private bucket", "website GET private bucket, owner-signed"] {
|
||||
let refused = answer(answers, name);
|
||||
assert_eq!(refused.status, 403, "{name}: {refused:?}");
|
||||
assert!(!refused.body_text().contains("home"), "{name} leaked the index: {refused:?}");
|
||||
}
|
||||
let signed_public = answer(answers, "website GET public bucket, owner-signed");
|
||||
assert_eq!((signed_public.status, signed_public.body_text().as_str()), (200, "home"));
|
||||
for name in ["ListObjectsV2 metadata=true", "ListObjectVersions metadata=true"] {
|
||||
let listing = answer(answers, name);
|
||||
let body = listing.body_text();
|
||||
assert_eq!(listing.status, 200, "{name}: {body}");
|
||||
assert!(body.contains("<UserMetadata>"), "{name} lost the extension: {body}");
|
||||
assert!(body.contains("blue"), "{name} lost the user metadata value: {body}");
|
||||
}
|
||||
for name in [
|
||||
"ListObjectsV2 metadata=true, no list permission",
|
||||
"ListObjectsV2 metadata=true, anonymous",
|
||||
] {
|
||||
let refused = answer(answers, name);
|
||||
assert_eq!(refused.status, 403, "{name}: {refused:?}");
|
||||
assert!(refused.body_text().contains("<Code>AccessDenied</Code>"), "{name}: {refused:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_gateway_stack_answers_website_and_metadata_listing_requests_as_the_legacy_stack_does() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let legacy_env = [("RUSTFS_S3_STACK", "legacy"), ("RUSTFS_WEBSITE_DOMAINS", WEBSITE_DOMAIN)];
|
||||
let gateway_env = [("RUSTFS_S3_STACK", "gateway"), ("RUSTFS_WEBSITE_DOMAINS", WEBSITE_DOMAIN)];
|
||||
|
||||
env.start_rustfs_server_with_env(vec![], &legacy_env).await?;
|
||||
seed(&env).await?;
|
||||
let legacy = observe(&env).await?;
|
||||
let legacy_plain = plain_listing(&env).await?;
|
||||
|
||||
env.restart_server_preserving_data(vec![], &gateway_env).await?;
|
||||
let gateway = observe(&env).await?;
|
||||
let gateway_plain = plain_listing(&env).await?;
|
||||
env.stop_server();
|
||||
|
||||
// The control: a request that is neither family is the gateway's own. Legacy lists the
|
||||
// bucket; the gateway stack does not serve ListObjectsV2 yet. Equal answers here would
|
||||
// mean the restart never reached the gateway stack, and every comparison below is void.
|
||||
assert_eq!(legacy_plain.status, 200, "{legacy_plain:?}");
|
||||
assert_eq!(gateway_plain.status, 501, "{gateway_plain:?}");
|
||||
|
||||
assert_is_a_website(&legacy);
|
||||
// Every difference, not the first one: a family that regressed alone must name itself.
|
||||
let differences: Vec<String> = legacy
|
||||
.iter()
|
||||
.zip(&gateway)
|
||||
.filter(|((_, on_legacy), (_, on_gateway))| on_legacy != on_gateway)
|
||||
.map(|((name, on_legacy), (_, on_gateway))| format!("{name}:\n legacy: {on_legacy:?}\n gateway: {on_gateway:?}"))
|
||||
.collect();
|
||||
assert!(
|
||||
differences.is_empty(),
|
||||
"the gateway stack answered {} of {} requests differently from the legacy stack:\n{}",
|
||||
differences.len(),
|
||||
legacy.len(),
|
||||
differences.join("\n")
|
||||
);
|
||||
assert_eq!(gateway.len(), legacy.len());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The metadata listing on a bucket that does not exist: answered by the legacy route on both
|
||||
/// stacks, so `NoSuchBucket`, never the gateway's `NotImplemented`.
|
||||
#[tokio::test]
|
||||
async fn a_metadata_listing_of_a_missing_bucket_is_refused_by_legacy_on_the_gateway_stack() -> TestResult {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_S3_STACK", "gateway")])
|
||||
.await?;
|
||||
let host = env.address.clone();
|
||||
let missing = signed(
|
||||
&env,
|
||||
&host,
|
||||
"/interim-no-such-bucket?list-type=2&metadata=true",
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await?;
|
||||
let plain = signed(&env, &host, "/interim-no-such-bucket?list-type=2", &env.access_key, &env.secret_key).await?;
|
||||
env.stop_server();
|
||||
|
||||
assert_eq!(missing.status, 404, "{missing:?}");
|
||||
assert!(missing.body_text().contains("<Code>NoSuchBucket</Code>"), "{missing:?}");
|
||||
assert_eq!(plain.status, 501, "{plain:?}");
|
||||
assert!(plain.body_text().contains("<Code>NotImplemented</Code>"), "{plain:?}");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,7 @@ license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "RustFS implementations of the rustfs-gateway extension traits (credentials and policy facts over IAM)."
|
||||
description = "RustFS implementations of the rustfs-gateway extension traits (credentials, policy facts, virtual hosts, bucket owner and request id)."
|
||||
keywords = ["rustfs", "gateway", "s3", "iam"]
|
||||
categories = ["web-programming", "authentication"]
|
||||
authors.workspace = true
|
||||
@@ -35,6 +35,7 @@ ring = 2
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
http = { workspace = true }
|
||||
rustfs-gateway = { workspace = true }
|
||||
rustfs-credentials = { workspace = true }
|
||||
rustfs-iam = { workspace = true }
|
||||
@@ -42,5 +43,6 @@ rustfs-policy = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
rustfs-gateway-http = { workspace = true }
|
||||
time = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt"] }
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway's [`BucketOwnerSource`]: the one owner RustFS reports, for a bucket that exists.
|
||||
//!
|
||||
//! Responsible for: confirming through a [`BucketLookup`] that a bucket exists, answering the
|
||||
//! single owner id the deployment reports ([`RustfsBucketOwners::new`]) for it, and turning every
|
||||
//! outcome that is not a usable owner into the gateway's one refusal, [`BucketOwnerError`].
|
||||
//! NOT responsible for: choosing the owner id (the binary passes the constant it already reports
|
||||
//! as the owner in listings), comparing it with `x-amz-expected-bucket-owner` (the gateway does,
|
||||
//! byte for byte), or authorizing anything.
|
||||
//! Upstream: the gateway's expected-owner check, which consults this source only when a request
|
||||
//! carries the header, after authorization allowed it. Downstream: the lookup the binary supplies
|
||||
//! over the ecstore bucket metadata system.
|
||||
//!
|
||||
//! # What an owner is in RustFS
|
||||
//!
|
||||
//! RustFS stores no per-bucket owner: bucket ACL writes are no-ops and nothing writes the ACL
|
||||
//! document, so the only owner RustFS reports anywhere (ListBuckets, ACL and multipart listings)
|
||||
//! is one constant. The source answers exactly that constant. A caller therefore passes the
|
||||
//! gateway's check only by presenting the owner id RustFS lists, which separates no tenants; it is
|
||||
//! a compatibility answer, and the constant's own documentation says it must not decide
|
||||
//! authorization.
|
||||
//!
|
||||
//! # Difference from legacy (ledger entry R-071)
|
||||
//!
|
||||
//! Legacy RustFS ignores `x-amz-expected-bucket-owner` entirely. On the gateway stack a request
|
||||
//! carrying it is served only when the value equals the reported owner id; otherwise it is `403
|
||||
//! AccessDenied`. A missing bucket and a backend failure with the header present are also `403`
|
||||
//! at the pinned gateway, where legacy answers `404` and `5xx`: [`BucketOwnerError`] has one value,
|
||||
//! so it cannot express either. [`RustfsBucketOwners::resolve`] keeps the two apart at this
|
||||
//! adapter's boundary so the gateway can surface them once it grows the classes.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::{BoxFuture, BucketName, BucketOwnerError, BucketOwnerSource};
|
||||
|
||||
/// Why a bucket yielded no owner.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum OwnerLookupError {
|
||||
/// The bucket does not exist.
|
||||
NoSuchBucket,
|
||||
/// The metadata backend failed, could not be reached, or the owner is not usable (an empty
|
||||
/// owner id is a misconfiguration), so the bucket's state is unknown.
|
||||
Backend,
|
||||
}
|
||||
|
||||
/// The narrow slice of RustFS bucket metadata an owner answer needs: whether a bucket exists.
|
||||
///
|
||||
/// Exists so the source can be tested against a fake. The binary implements it over the ecstore
|
||||
/// bucket metadata system, mapping `BucketNotFound` to [`OwnerLookupError::NoSuchBucket`] and every
|
||||
/// other error to [`OwnerLookupError::Backend`].
|
||||
pub trait BucketLookup: Send + Sync + 'static {
|
||||
/// `Ok(())` when `bucket` exists.
|
||||
fn check<'a>(&'a self, bucket: &'a str) -> BoxFuture<'a, Result<(), OwnerLookupError>>;
|
||||
}
|
||||
|
||||
/// A [`BucketOwnerSource`] answering one owner id for every bucket that exists.
|
||||
pub struct RustfsBucketOwners<L> {
|
||||
lookup: L,
|
||||
owner: Arc<str>,
|
||||
}
|
||||
|
||||
impl<L: BucketLookup> RustfsBucketOwners<L> {
|
||||
/// A source answering `owner` for every bucket `lookup` finds.
|
||||
///
|
||||
/// An empty `owner` is no owner: every answer is then [`OwnerLookupError::Backend`].
|
||||
#[must_use]
|
||||
pub fn new(lookup: L, owner: impl Into<Arc<str>>) -> Self {
|
||||
Self {
|
||||
lookup,
|
||||
owner: owner.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The owner of `bucket`, or why there is none. Missing and failing stay distinct here.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`OwnerLookupError::NoSuchBucket`] when the bucket does not exist;
|
||||
/// [`OwnerLookupError::Backend`] on any lookup failure or when the configured owner is empty.
|
||||
pub async fn resolve(&self, bucket: &str) -> Result<Arc<str>, OwnerLookupError> {
|
||||
self.lookup.check(bucket).await?;
|
||||
if self.owner.is_empty() {
|
||||
return Err(OwnerLookupError::Backend);
|
||||
}
|
||||
Ok(Arc::clone(&self.owner))
|
||||
}
|
||||
}
|
||||
|
||||
impl<L: BucketLookup> BucketOwnerSource for RustfsBucketOwners<L> {
|
||||
fn owner<'a>(&'a self, bucket: &'a BucketName) -> BoxFuture<'a, Result<Arc<str>, BucketOwnerError>> {
|
||||
Box::pin(async move {
|
||||
// The pinned gateway collapses both classes (R-071), so both are the one refusal.
|
||||
self.resolve(bucket.as_str())
|
||||
.await
|
||||
.map_err(|_: OwnerLookupError| BucketOwnerError::unavailable())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
|
||||
/// Stands for the owner id RustFS reports; the binary passes its own constant.
|
||||
const OWNER: &str = "reported-owner-id";
|
||||
|
||||
struct Fake {
|
||||
answer: Result<(), OwnerLookupError>,
|
||||
asked: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
impl Fake {
|
||||
fn answering(answer: Result<(), OwnerLookupError>) -> Self {
|
||||
Self {
|
||||
answer,
|
||||
asked: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketLookup for Fake {
|
||||
fn check<'a>(&'a self, bucket: &'a str) -> BoxFuture<'a, Result<(), OwnerLookupError>> {
|
||||
self.asked.lock().unwrap().push(bucket.to_owned());
|
||||
let answer = self.answer;
|
||||
Box::pin(async move { answer })
|
||||
}
|
||||
}
|
||||
|
||||
fn source(answer: Result<(), OwnerLookupError>) -> RustfsBucketOwners<Fake> {
|
||||
RustfsBucketOwners::new(Fake::answering(answer), OWNER)
|
||||
}
|
||||
|
||||
async fn gateway_answer(answer: Result<(), OwnerLookupError>) -> Result<Arc<str>, BucketOwnerError> {
|
||||
source(answer).owner(&BucketName::new("photos").expect("a bucket name")).await
|
||||
}
|
||||
|
||||
/// Positive: an existing bucket's owner is the configured id, byte for byte.
|
||||
#[tokio::test]
|
||||
async fn an_existing_bucket_has_the_configured_owner() {
|
||||
assert_eq!(gateway_answer(Ok(())).await.as_deref(), Ok(OWNER));
|
||||
}
|
||||
|
||||
/// Positive: every bucket gets the same owner, since RustFS stores none per bucket.
|
||||
#[tokio::test]
|
||||
async fn every_bucket_has_the_same_owner() {
|
||||
let source = source(Ok(()));
|
||||
assert_eq!(source.resolve("photos").await.as_deref(), Ok(OWNER));
|
||||
assert_eq!(source.resolve("backups").await.as_deref(), Ok(OWNER));
|
||||
}
|
||||
|
||||
/// Positive: the lookup is asked about exactly the bucket the gateway named.
|
||||
#[tokio::test]
|
||||
async fn the_lookup_is_asked_about_the_named_bucket() {
|
||||
let source = source(Ok(()));
|
||||
source
|
||||
.owner(&BucketName::new("photos").expect("a bucket name"))
|
||||
.await
|
||||
.expect("an owner");
|
||||
assert_eq!(*source.lookup.asked.lock().unwrap(), vec!["photos".to_owned()]);
|
||||
}
|
||||
|
||||
/// Negative: a missing bucket is reported as missing at the adapter boundary.
|
||||
#[tokio::test]
|
||||
async fn a_missing_bucket_is_reported_as_missing() {
|
||||
assert_eq!(
|
||||
source(Err(OwnerLookupError::NoSuchBucket)).resolve("photos").await,
|
||||
Err(OwnerLookupError::NoSuchBucket)
|
||||
);
|
||||
}
|
||||
|
||||
/// Negative: a failing backend is reported as a failure, never as an owner and never as missing.
|
||||
#[tokio::test]
|
||||
async fn a_backend_failure_is_reported_as_a_failure() {
|
||||
assert_eq!(
|
||||
source(Err(OwnerLookupError::Backend)).resolve("photos").await,
|
||||
Err(OwnerLookupError::Backend)
|
||||
);
|
||||
}
|
||||
|
||||
/// Negative: the pinned gateway sees one refusal for both, and never an owner.
|
||||
#[tokio::test]
|
||||
async fn the_gateway_is_refused_in_both_cases() {
|
||||
assert_eq!(
|
||||
gateway_answer(Err(OwnerLookupError::NoSuchBucket)).await,
|
||||
Err(BucketOwnerError::unavailable())
|
||||
);
|
||||
assert_eq!(gateway_answer(Err(OwnerLookupError::Backend)).await, Err(BucketOwnerError::unavailable()));
|
||||
}
|
||||
|
||||
/// Negative: an empty configured owner is no owner; it must not equal an empty assertion.
|
||||
#[tokio::test]
|
||||
async fn an_empty_owner_is_refused() {
|
||||
let source = RustfsBucketOwners::new(Fake::answering(Ok(())), "");
|
||||
assert_eq!(source.resolve("photos").await, Err(OwnerLookupError::Backend));
|
||||
assert_eq!(
|
||||
source.owner(&BucketName::new("photos").expect("a bucket name")).await,
|
||||
Err(BucketOwnerError::unavailable())
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway's [`HostResolver`], built from RustFS's configured domain list.
|
||||
//!
|
||||
//! Responsible for: turning the `RUSTFS_SERVER_DOMAINS` list into the gateway's legacy RustFS
|
||||
//! virtual-host resolver.
|
||||
//! NOT responsible for: parsing a `Host` header, matching a domain, validating bucket names (all
|
||||
//! done by [`LegacyRustfsVirtualHosts`]), reading the environment, or deciding that a console
|
||||
//! deployment serves no domains (the caller passes an empty list then, as `s3_host_domains` in
|
||||
//! `rustfs/src/server/http.rs` returns `None`). An empty list needs no branch here:
|
||||
//! [`LegacyRustfsVirtualHosts`] built from no domain reads every host path-style, exactly as the
|
||||
//! gateway's `PathStyleOnly` does.
|
||||
//! Upstream: the RustFS configuration (`Config::server_domains`). Downstream: the gateway's
|
||||
//! `ServiceBuilder`, which holds the resolver as `Arc<dyn HostResolver>`.
|
||||
//!
|
||||
//! # What "the same as `MultiDomain`" means
|
||||
//!
|
||||
//! RustFS today hands the same list to the s3s framework's multi-domain host, which ignores ports
|
||||
//! on both sides when matching a domain, takes the whole prefix in front of a matched domain as
|
||||
//! the bucket, compares a domain exactly (so a host is matched on the case it arrives in), and,
|
||||
//! for a host no domain matches, falls back to CNAME-style: a host that is itself a bucket name
|
||||
//! (lower-cased) is that bucket. [`LegacyRustfsVirtualHosts`] reproduces exactly that.
|
||||
//!
|
||||
//! What reads **path-style**: an empty domain list; a configured domain itself, with or without a
|
||||
//! port; an IP address, or a socket address (`10.0.0.1`, `[::1]:9000`); a host that is a valid
|
||||
//! domain but, lower-cased, is no bucket name (it carries a port, is over 63 bytes, or starts
|
||||
//! with `xn--`). What reads as a **bucket**: anything under a configured domain, and any other
|
||||
//! bucket-shaped host, which is a bucket of its own (`localhost`, `badexample.com`, `EXAMPLE.COM`).
|
||||
//! What is **refused** before routing: a host that is no domain (`under_score.org`, a bare
|
||||
//! `[::1]`, a trailing dot), and a bucket under a domain that the bucket rules refuse
|
||||
//! (`PHOTOS.example.com`, `ab.example.com`). The tests below pin each of those readings.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::{HostResolver, LegacyDomainError, LegacyRustfsVirtualHosts};
|
||||
|
||||
/// Builds the host resolver for RustFS's configured domain list.
|
||||
///
|
||||
/// An empty list reads every host path-style. Otherwise the list is read as legacy RustFS reads
|
||||
/// it: a later entry whose name without its port repeats an earlier one is dropped, and a list
|
||||
/// that legacy RustFS refuses at start-up is refused here.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`LegacyDomainError`] for an entry that is not a domain, or for two entries where one equals or
|
||||
/// is a subdomain of the other.
|
||||
pub fn rustfs_host_resolver<I, S>(domains: I) -> Result<Arc<dyn HostResolver>, LegacyDomainError>
|
||||
where
|
||||
I: IntoIterator<Item = S>,
|
||||
S: AsRef<str>,
|
||||
{
|
||||
Ok(Arc::new(LegacyRustfsVirtualHosts::new(domains)?))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::Method;
|
||||
use rustfs_gateway::{Addressing, HostQuery, HostRefusal, TargetKind};
|
||||
|
||||
/// What the resolver reads `GET path` with `Host: host` as: the host-named bucket (`None` for
|
||||
/// path style), the routed target, and the refusal, if any.
|
||||
fn read(resolver: &dyn HostResolver, path: &str, host: &str) -> (Option<String>, TargetKind, Option<HostRefusal>) {
|
||||
let method = Method::GET;
|
||||
let request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(path)
|
||||
.header("host", host)
|
||||
.body(())
|
||||
.expect("a valid request");
|
||||
let effective = rustfs_gateway_http::effective_host(&request).expect("an acceptable host");
|
||||
let query = HostQuery {
|
||||
host: &effective,
|
||||
path,
|
||||
method: &method,
|
||||
};
|
||||
let resolved = resolver.resolve(&query);
|
||||
let bucket = match &resolved.addressing {
|
||||
Addressing::VirtualHosted { bucket, .. } => Some(bucket.as_str().to_owned()),
|
||||
Addressing::Path => None,
|
||||
};
|
||||
(bucket, resolved.target, resolver.refusal(&query))
|
||||
}
|
||||
|
||||
fn with_domains(domains: &[&str]) -> Arc<dyn HostResolver> {
|
||||
rustfs_host_resolver(domains).expect("a list RustFS accepts")
|
||||
}
|
||||
|
||||
fn bucket(domains: &[&str], host: &str) -> Option<String> {
|
||||
read(&*with_domains(domains), "/key", host).0
|
||||
}
|
||||
|
||||
/// Positive: a host under a configured domain names the bucket in front of it.
|
||||
#[test]
|
||||
fn host_under_domain_names_the_bucket() {
|
||||
assert_eq!(
|
||||
read(&*with_domains(&["example.com"]), "/key", "photos.example.com"),
|
||||
(Some("photos".to_owned()), TargetKind::Object, None)
|
||||
);
|
||||
assert_eq!(read(&*with_domains(&["example.com"]), "/", "photos.example.com").1, TargetKind::Bucket);
|
||||
}
|
||||
|
||||
/// Positive: the port on the host, or on the configured domain, does not take part in matching.
|
||||
#[test]
|
||||
fn ports_are_ignored_on_both_sides() {
|
||||
assert_eq!(bucket(&["example.com"], "photos.example.com:9000").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&["example.com:9000"], "photos.example.com").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&["example.com:9000"], "photos.example.com:1234").as_deref(), Some("photos"));
|
||||
}
|
||||
|
||||
/// Positive: the whole prefix in front of the domain is the bucket, dots kept.
|
||||
#[test]
|
||||
fn whole_prefix_is_the_bucket() {
|
||||
assert_eq!(bucket(&["example.com"], "my.dotted.bkt.example.com").as_deref(), Some("my.dotted.bkt"));
|
||||
}
|
||||
|
||||
/// Positive: with several domains, each one resolves its own subdomains.
|
||||
#[test]
|
||||
fn every_configured_domain_resolves() {
|
||||
let domains = ["example.com", "s3.local"];
|
||||
assert_eq!(bucket(&domains, "abc.example.com").as_deref(), Some("abc"));
|
||||
assert_eq!(bucket(&domains, "xyz.s3.local").as_deref(), Some("xyz"));
|
||||
}
|
||||
|
||||
/// Positive: the domain itself is the endpoint, so the bucket comes from the path.
|
||||
#[test]
|
||||
fn the_domain_itself_is_path_style() {
|
||||
assert_eq!(bucket(&["example.com"], "example.com"), None);
|
||||
assert_eq!(bucket(&["example.com"], "example.com:9000"), None);
|
||||
}
|
||||
|
||||
/// Negative: no configured domain means path style only, for every host shape.
|
||||
#[test]
|
||||
fn no_domain_is_path_style_only() {
|
||||
for host in ["photos.example.com", "localhost", "localhost:9000", "10.0.0.1"] {
|
||||
assert_eq!(bucket(&[], host), None, "{host}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative: an address, and a host carrying a port that no domain matches, are path style.
|
||||
#[test]
|
||||
fn addresses_and_unmatched_ports_are_path_style() {
|
||||
for host in ["10.0.0.1", "10.0.0.1:9000", "localhost:9000", "other.org:9000"] {
|
||||
assert_eq!(bucket(&["example.com"], host), None, "{host}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative: a host that only ends in the same letters as a domain is not under it. It is not
|
||||
/// stripped to `bad`; the only reading it gets is the CNAME-style one, the whole host.
|
||||
#[test]
|
||||
fn suffix_lookalike_is_not_under_the_domain() {
|
||||
assert_eq!(bucket(&["example.com"], "badexample.com").as_deref(), Some("badexample.com"));
|
||||
assert_eq!(bucket(&["example.com"], "example.com.evil.org").as_deref(), Some("example.com.evil.org"));
|
||||
}
|
||||
|
||||
/// Negative: matching a configured domain is exact on case, as the legacy multi-domain host does it, so a
|
||||
/// mixed-case host is not stripped to its prefix; it is the lower-cased whole host.
|
||||
#[test]
|
||||
fn mixed_case_host_is_not_stripped_to_a_prefix() {
|
||||
assert_eq!(bucket(&["example.com"], "Photos.Example.COM").as_deref(), Some("photos.example.com"));
|
||||
}
|
||||
|
||||
/// Negative: a host that is no domain at all is refused, not routed.
|
||||
#[test]
|
||||
fn a_host_that_is_no_domain_is_refused() {
|
||||
let resolver = with_domains(&["example.com"]);
|
||||
assert_eq!(read(&*resolver, "/k", "under_score.org").2, Some(HostRefusal::UnusableHost));
|
||||
}
|
||||
|
||||
/// Negative: a bucket the host names that bucket rules refuse is refused.
|
||||
#[test]
|
||||
fn a_host_named_bucket_the_rules_refuse_is_refused() {
|
||||
let resolver = with_domains(&["example.com"]);
|
||||
assert_eq!(read(&*resolver, "/k", "ab.example.com").2, Some(HostRefusal::RefusedBucket));
|
||||
assert_eq!(read(&*resolver, "/k", "bad_host.example.com").2, Some(HostRefusal::RefusedBucket));
|
||||
}
|
||||
|
||||
/// Negative: a list legacy RustFS refuses at start-up is refused.
|
||||
#[test]
|
||||
fn domain_lists_legacy_rustfs_refuses_are_refused() {
|
||||
for list in [
|
||||
vec!["example.com", "a.example.com"],
|
||||
vec!["example.com", "com"],
|
||||
vec!["bad_domain"],
|
||||
vec![""],
|
||||
vec!["example.com:notaport"],
|
||||
] {
|
||||
assert!(rustfs_host_resolver(&list).is_err(), "{list:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive: a later repeat of a name, whatever its port, is dropped rather than refused, as
|
||||
/// `s3_host_domains` drops it before the legacy multi-domain host sees the list.
|
||||
#[test]
|
||||
fn a_repeat_of_a_domain_is_dropped() {
|
||||
assert_eq!(
|
||||
bucket(&["example.com:9411", "example.com:9000"], "xyz.example.com").as_deref(),
|
||||
Some("xyz")
|
||||
);
|
||||
}
|
||||
|
||||
/// The refusal alone, for hosts whose bucket reading is not the point.
|
||||
fn refusal(domains: &[&str], host: &str) -> Option<HostRefusal> {
|
||||
read(&*with_domains(domains), "/k", host).2
|
||||
}
|
||||
|
||||
/// Positive: a bucket-shaped host no domain matches is a bucket of its own, `localhost` and
|
||||
/// an upper-case spelling of the domain among them.
|
||||
#[test]
|
||||
fn bucket_shaped_hosts_outside_every_domain_are_their_own_bucket() {
|
||||
assert_eq!(bucket(&["example.com"], "localhost").as_deref(), Some("localhost"));
|
||||
assert_eq!(bucket(&["example.com"], "EXAMPLE.COM").as_deref(), Some("example.com"));
|
||||
assert_eq!(bucket(&["example.com"], "cdn.other.org").as_deref(), Some("cdn.other.org"));
|
||||
}
|
||||
|
||||
/// Negative: a bucket the rules refuse under a domain is refused, and reads no bucket.
|
||||
#[test]
|
||||
fn an_upper_case_bucket_under_the_domain_is_refused() {
|
||||
assert_eq!(bucket(&["example.com"], "PHOTOS.example.com"), None);
|
||||
assert_eq!(refusal(&["example.com"], "PHOTOS.example.com"), Some(HostRefusal::RefusedBucket));
|
||||
}
|
||||
|
||||
/// Negative: an IPv6 literal. The bare form is no domain and is refused; with a port it is a
|
||||
/// socket address and reads path-style.
|
||||
#[test]
|
||||
fn ipv6_literals_follow_the_address_rules() {
|
||||
assert_eq!(refusal(&["example.com"], "[::1]"), Some(HostRefusal::UnusableHost));
|
||||
assert_eq!(bucket(&["example.com"], "[::1]:9000"), None);
|
||||
assert_eq!(refusal(&["example.com"], "[::1]:9000"), None);
|
||||
}
|
||||
|
||||
/// Negative: a trailing root dot makes the host no domain, so it is refused, not stripped.
|
||||
#[test]
|
||||
fn a_trailing_dot_is_refused() {
|
||||
assert_eq!(refusal(&["example.com"], "photos.example.com."), Some(HostRefusal::UnusableHost));
|
||||
assert_eq!(bucket(&["example.com"], "photos.example.com."), None);
|
||||
}
|
||||
|
||||
/// Negative: an empty label never reaches the resolver; the gateway's wire layer refuses the
|
||||
/// host first (legacy answers 400 as well).
|
||||
#[test]
|
||||
fn an_empty_label_is_refused_before_resolution() {
|
||||
for host in ["a..b.example.com", ".example.com"] {
|
||||
let request = http::Request::builder()
|
||||
.uri("/k")
|
||||
.header("host", host)
|
||||
.body(())
|
||||
.expect("a valid request");
|
||||
assert!(rustfs_gateway_http::effective_host(&request).is_err(), "{host}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative: a punycode label is no bucket under a domain; outside every domain the host is
|
||||
/// a valid domain that is no bucket name, so it reads path-style.
|
||||
#[test]
|
||||
fn a_punycode_label_is_never_a_bucket() {
|
||||
assert_eq!(refusal(&["example.com"], "xn--bcher-kva.example.com"), Some(HostRefusal::RefusedBucket));
|
||||
assert_eq!(bucket(&["example.com"], "xn--bcher-kva.de"), None);
|
||||
assert_eq!(refusal(&["example.com"], "xn--bcher-kva.de"), None);
|
||||
}
|
||||
|
||||
/// Negative: a label over 63 bytes is no bucket, under a domain or outside one.
|
||||
#[test]
|
||||
fn a_label_over_63_bytes_is_never_a_bucket() {
|
||||
let long = "a".repeat(64);
|
||||
let under = format!("{long}.example.com");
|
||||
assert_eq!(refusal(&["example.com"], &under), Some(HostRefusal::RefusedBucket));
|
||||
let outside = format!("{long}.org");
|
||||
assert_eq!(bucket(&["example.com"], &outside), None);
|
||||
assert_eq!(refusal(&["example.com"], &outside), None);
|
||||
let at_limit = "a".repeat(63);
|
||||
assert_eq!(bucket(&["example.com"], &format!("{at_limit}.example.com")), Some(at_limit));
|
||||
}
|
||||
|
||||
/// Positive and negative: two configured domains where one merely ends in the other's letters
|
||||
/// both work, and each domain itself is path-style.
|
||||
#[test]
|
||||
fn a_domain_and_its_suffix_lookalike_are_both_served() {
|
||||
let domains = ["example.com", "badexample.com"];
|
||||
assert_eq!(bucket(&domains, "photos.example.com").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&domains, "photos.badexample.com").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&domains, "badexample.com"), None);
|
||||
assert_eq!(bucket(&domains, "bad.example.com").as_deref(), Some("bad"));
|
||||
}
|
||||
|
||||
/// Negative: a configured domain is matched exactly, case included.
|
||||
#[test]
|
||||
fn a_configured_domain_is_matched_on_its_case() {
|
||||
assert_eq!(bucket(&["Example.COM"], "photos.Example.COM").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&["Example.COM"], "photos.example.com").as_deref(), Some("photos.example.com"));
|
||||
assert_eq!(bucket(&["Example.COM"], "Example.COM"), None);
|
||||
}
|
||||
}
|
||||
@@ -18,17 +18,23 @@
|
||||
//! gateway depends on it. Each module adapts one extension point to the RustFS subsystem that
|
||||
//! owns the answer.
|
||||
|
||||
pub mod bucket_owner;
|
||||
pub mod credentials;
|
||||
pub mod host;
|
||||
pub mod policy;
|
||||
pub mod root;
|
||||
pub mod trace;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::{Authenticator, CredentialProvider, ServiceBuilder};
|
||||
|
||||
pub use bucket_owner::{BucketLookup, OwnerLookupError, RustfsBucketOwners};
|
||||
pub use credentials::{IamCredentials, IamKeyLookup, STS_SESSION_ISSUER};
|
||||
pub use host::rustfs_host_resolver;
|
||||
pub use policy::{IamPolicySource, IamPrincipalLookup, PrincipalKind, RustfsPrincipal};
|
||||
pub use root::{MIN_SECRET_KEY_LEN, RootCredentials, RootCredentialsError, RootSource};
|
||||
pub use trace::{AttachOutcome, RequestIdCarrier, attach_host_request_id, host_request_id};
|
||||
|
||||
/// The IAM-backed credential provider and policy source, assembled together.
|
||||
///
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The request id RustFS's outer layer minted, handed to the gateway as its own when that is safe.
|
||||
//!
|
||||
//! Responsible for: reading the id from the context the outer layer put in the request's
|
||||
//! extensions ([`RequestIdCarrier`]) and, when it is inside the gateway's identifier alphabet and
|
||||
//! length, inserting it as the gateway's [`HostRequestId`].
|
||||
//! NOT responsible for: minting any id, refusing any request (nothing here can fail a request),
|
||||
//! choosing the response headers (the gateway writes `x-amz-request-id` from the id it was
|
||||
//! handed, and RustFS's outer layer overwrites that header on S3 paths), or the layer that mints
|
||||
//! (`rustfs/src/server/layer.rs`). It never reads a header itself.
|
||||
//! Upstream: RustFS's `RequestContext`, which the binary makes a [`RequestIdCarrier`]. Downstream:
|
||||
//! the gateway service, which reads the [`HostRequestId`] extension once, at the top.
|
||||
//!
|
||||
//! # Whose id it is
|
||||
//!
|
||||
//! On S3 paths the outer layer mints a server-owned UUID. On admin, STS and console paths the
|
||||
//! context copies the caller's `x-request-id` (or `x-amz-request-id`), as legacy does, so there
|
||||
//! the id can be caller-chosen. Either way the id is attached only when it is inert, that is 1 to
|
||||
//! 64 bytes of ASCII letters, digits, `-`, `.`, `_` and `~`, which is safe in a header, an XML
|
||||
//! document and a log line. Anything else is not attached and the request is **forwarded
|
||||
//! unchanged**: the gateway then mints its own id. No request is ever refused because of its id.
|
||||
//!
|
||||
//! # Known difference when nothing is attached
|
||||
//!
|
||||
//! The gateway answers with the id it minted (error body `<RequestId>`, its events and audit
|
||||
//! records) while, on S3 paths, the outer layer overwrites the response header with RustFS's id.
|
||||
//! So for such a request the header and the error body name different ids. That needs a caller-
|
||||
//! chosen id outside the alphabet, or a request that never passed the outer layer.
|
||||
//!
|
||||
//! # Why this is not a `TraceSource`
|
||||
//!
|
||||
//! The gateway's `TraceSource::mint` takes no request, so a source cannot read the id RustFS
|
||||
//! already minted; the gateway's documented path for a host that identifies its own requests is
|
||||
//! the [`HostRequestId`] extension, and this module fills it.
|
||||
//!
|
||||
//! # Why a carrier trait
|
||||
//!
|
||||
//! `RequestContext` lives in the `rustfs` binary crate, which a library crate cannot depend on.
|
||||
//! The binary implements [`RequestIdCarrier`] for it, and this module looks it up in the
|
||||
//! extensions by that type.
|
||||
|
||||
use rustfs_gateway::HostRequestId;
|
||||
|
||||
/// A type in the request extensions that holds the id the outer layer minted or propagated.
|
||||
pub trait RequestIdCarrier: Send + Sync + 'static {
|
||||
/// The canonical request id: the value RustFS logs and the response should carry.
|
||||
fn request_id(&self) -> &str;
|
||||
}
|
||||
|
||||
/// What [`attach_host_request_id`] did. Ignoring it is harmless: every outcome forwards the request.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum AttachOutcome {
|
||||
/// The carrier's id was inserted as the request's [`HostRequestId`].
|
||||
Attached,
|
||||
/// The request carries no context from the outer layer; nothing was inserted.
|
||||
NoContext,
|
||||
/// The carrier's id is empty, too long, or outside the gateway's alphabet; nothing was
|
||||
/// inserted and the id was not repaired.
|
||||
UnusableId,
|
||||
}
|
||||
|
||||
/// The gateway's [`HostRequestId`] for the id `carrier` holds, or `None` when it is not inert.
|
||||
#[must_use]
|
||||
pub fn host_request_id<C: RequestIdCarrier + ?Sized>(carrier: &C) -> Option<HostRequestId> {
|
||||
HostRequestId::new(carrier.request_id()).ok()
|
||||
}
|
||||
|
||||
/// Reads the `C` the outer layer put in `request`'s extensions and, when its id is inert, inserts
|
||||
/// it as the request's [`HostRequestId`].
|
||||
///
|
||||
/// Never refuses the request: the caller forwards it whatever the outcome.
|
||||
pub fn attach_host_request_id<C: RequestIdCarrier, B>(request: &mut http::Request<B>) -> AttachOutcome {
|
||||
let Some(carrier) = request.extensions().get::<C>() else {
|
||||
return AttachOutcome::NoContext;
|
||||
};
|
||||
let Some(id) = host_request_id(carrier) else {
|
||||
return AttachOutcome::UnusableId;
|
||||
};
|
||||
request.extensions_mut().insert(id);
|
||||
AttachOutcome::Attached
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Ctx(String);
|
||||
|
||||
impl RequestIdCarrier for Ctx {
|
||||
fn request_id(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
const UUID: &str = "7c9e6679-7425-40de-944b-e07fc1f90ae7";
|
||||
|
||||
fn request_with(id: &str) -> http::Request<()> {
|
||||
let mut request = http::Request::new(());
|
||||
request.extensions_mut().insert(Ctx(id.to_owned()));
|
||||
request
|
||||
}
|
||||
|
||||
fn handed_over(request: &http::Request<()>) -> Option<String> {
|
||||
request
|
||||
.extensions()
|
||||
.get::<HostRequestId>()
|
||||
.map(|id| id.request_id().as_str().to_owned())
|
||||
}
|
||||
|
||||
fn attach(request: &mut http::Request<()>) -> AttachOutcome {
|
||||
attach_host_request_id::<Ctx, _>(request)
|
||||
}
|
||||
|
||||
/// Positive: the id RustFS minted is the id the gateway is handed, byte for byte.
|
||||
#[test]
|
||||
fn the_minted_id_is_handed_over_unchanged() {
|
||||
let mut request = request_with(UUID);
|
||||
assert_eq!(attach(&mut request), AttachOutcome::Attached);
|
||||
assert_eq!(handed_over(&request).as_deref(), Some(UUID));
|
||||
}
|
||||
|
||||
/// Positive: every shape RustFS mints is accepted, including its two fallbacks.
|
||||
#[test]
|
||||
fn every_shape_rustfs_mints_is_accepted() {
|
||||
for id in [UUID, "trace-0123456789abcdef", "req-0123456789abcdef"] {
|
||||
let mut request = request_with(id);
|
||||
assert_eq!(attach(&mut request), AttachOutcome::Attached, "{id}");
|
||||
assert_eq!(handed_over(&request).as_deref(), Some(id));
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive: a caller-supplied id that is inert (admin and STS paths copy it) is attached, and
|
||||
/// the longest allowed one is accepted.
|
||||
#[test]
|
||||
fn an_inert_caller_supplied_id_is_attached() {
|
||||
let longest = "a".repeat(64);
|
||||
for id in ["abc", "my.trace_id~1-2", longest.as_str()] {
|
||||
let mut request = request_with(id);
|
||||
assert_eq!(attach(&mut request), AttachOutcome::Attached, "{id}");
|
||||
assert_eq!(handed_over(&request).as_deref(), Some(id));
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive: two requests keep their own ids; nothing is shared or minted between them.
|
||||
#[test]
|
||||
fn each_request_keeps_its_own_id() {
|
||||
let mut first = request_with("first-id");
|
||||
let mut second = request_with("second-id");
|
||||
attach(&mut first);
|
||||
attach(&mut second);
|
||||
assert_eq!(handed_over(&first).as_deref(), Some("first-id"));
|
||||
assert_eq!(handed_over(&second).as_deref(), Some("second-id"));
|
||||
}
|
||||
|
||||
/// Positive: with both a carrier and an `x-amz-request-id` header, the carrier's id wins.
|
||||
#[test]
|
||||
fn the_carrier_wins_over_a_request_id_header() {
|
||||
let mut request = http::Request::builder()
|
||||
.header("x-amz-request-id", "caller-chosen")
|
||||
.header("x-request-id", "caller-chosen")
|
||||
.body(())
|
||||
.expect("a request");
|
||||
request.extensions_mut().insert(Ctx(UUID.to_owned()));
|
||||
assert_eq!(attach(&mut request), AttachOutcome::Attached);
|
||||
assert_eq!(handed_over(&request).as_deref(), Some(UUID));
|
||||
}
|
||||
|
||||
/// Negative: a request the outer layer never saw gets no id, and the request is not refused.
|
||||
#[test]
|
||||
fn a_missing_context_hands_over_nothing() {
|
||||
let mut request = http::Request::new(());
|
||||
assert_eq!(attach(&mut request), AttachOutcome::NoContext);
|
||||
assert_eq!(handed_over(&request), None);
|
||||
}
|
||||
|
||||
/// Negative: a header named like the id is not the id; only the context is read.
|
||||
#[test]
|
||||
fn a_request_id_header_alone_is_not_read() {
|
||||
let mut request = http::Request::builder()
|
||||
.header("x-amz-request-id", "caller-chosen")
|
||||
.header("x-request-id", "caller-chosen")
|
||||
.body(())
|
||||
.expect("a request");
|
||||
assert_eq!(attach(&mut request), AttachOutcome::NoContext);
|
||||
assert_eq!(handed_over(&request), None);
|
||||
}
|
||||
|
||||
/// Negative: an id outside the alphabet or the length is never attached or repaired, and the
|
||||
/// outcome says so rather than failing the request.
|
||||
#[test]
|
||||
fn an_id_the_gateway_cannot_carry_is_not_attached() {
|
||||
let too_long = "a".repeat(65);
|
||||
for id in [
|
||||
"",
|
||||
"has space",
|
||||
"line\r\nbreak",
|
||||
"<RequestId>",
|
||||
"quo\"te",
|
||||
"Root=1-5759e988",
|
||||
too_long.as_str(),
|
||||
] {
|
||||
let mut request = request_with(id);
|
||||
assert_eq!(attach(&mut request), AttachOutcome::UnusableId, "{id:?}");
|
||||
assert_eq!(handed_over(&request), None, "{id:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative: the unusable id is not repaired into a shorter or cleaned-up one.
|
||||
#[test]
|
||||
fn host_request_id_does_not_repair() {
|
||||
assert!(host_request_id(&Ctx("has space".to_owned())).is_none());
|
||||
assert!(host_request_id(&Ctx(format!("{UUID} "))).is_none());
|
||||
assert!(host_request_id(&Ctx(UUID.to_owned())).is_some());
|
||||
}
|
||||
}
|
||||
@@ -54,7 +54,6 @@ The four site-replication state-writer concurrency proofs reserve the available
|
||||
| PR, non-doc change | `Typos` | `ci.yml` `typos` | Via aggregate | `typos` |
|
||||
| Full selection | `Workspace tests` (serial ILM step) | `ci.yml` `workspace-tests` | Via aggregate | exact `Run ignored ILM integration tests serially` command in the job |
|
||||
| Nightly schedule, manual dispatch | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)`, `Test and Lint (http3)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo nextest run` with the job's `--features` |
|
||||
| Full selection, every event | `Test and Lint (gateway-stack)` | `ci.yml` `test-and-lint-gateway-stack` | Via aggregate | `cargo clippy -p rustfs --all-targets --features gateway-stack -- -D warnings`; `cargo nextest run -p rustfs --features gateway-stack` with the job's filter |
|
||||
| Nightly schedule, manual dispatch, full selection | `Connect Feature Boundaries` | `ci.yml` `connect-feature-boundaries` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail; `scripts/check_offline_enrollment_e2e.sh` |
|
||||
| PR, non-doc change | `Build RustFS Debug Binary` | `ci.yml` `build-rustfs-debug-binary` | Via aggregate; prerequisite for black-box jobs | `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks` (binary plus its `rustfs.e2e.json` sidecar) |
|
||||
| Nightly schedule, manual dispatch, full selection | `io_uring Integration (real)` | `ci.yml` `uring-integration` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
|
||||
|
||||
+1
-16
@@ -56,12 +56,6 @@ required-features = ["swift"]
|
||||
name = "swift_object_integration_test"
|
||||
required-features = ["swift"]
|
||||
|
||||
# Starts an embedded server on the gateway stack, which only a build with the
|
||||
# feature can select.
|
||||
[[test]]
|
||||
name = "embedded_gateway_stack_test"
|
||||
required-features = ["gateway-stack"]
|
||||
|
||||
[features]
|
||||
default = ["ftps", "webdav", "gcs"]
|
||||
fips = ["rustls/fips", "tokio-rustls/fips", "rustfs-tls-runtime/fips", "rustfs-ecstore/fips", "rustfs-protocols/fips", "rustfs-s3-client/fips", "rustfs-targets/fips", "rustfs-utils/fips"]
|
||||
@@ -75,19 +69,12 @@ license = []
|
||||
io-scheduler-debug = [] # Enable debug information in I/O scheduler
|
||||
tracing-chunk-debug = [] # Enable per-chunk tracing in data plane (high noise, for debugging only)
|
||||
full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope", "gcs"]
|
||||
# The debug-only E2E binary also carries the gateway S3 stack, so the gateway smoke
|
||||
# lane runs the same artifact as the legacy one with RUSTFS_S3_STACK as its only change.
|
||||
e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hooks", "gateway-stack"]
|
||||
e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hooks"]
|
||||
# Shortens Connect credentials only in debug E2E builds.
|
||||
connect-e2e-short-credentials = []
|
||||
# Builds the dedicated rustfs-cli-e2e target with a build-time public enrollment root.
|
||||
offline-enrollment-e2e-root = []
|
||||
rio-v2 = ["rustfs-ecstore/rio-v2"]
|
||||
# The rustfs-gateway S3 stack, selected at startup with RUSTFS_S3_STACK=gateway
|
||||
# (rustfs/backlog#2734). Under construction: off by default and never enabled in a
|
||||
# release build; the stack refuses to compile without debug assertions, and
|
||||
# `scripts/check_gateway_stack_release_boundary.sh` keeps it out of release inputs.
|
||||
gateway-stack = ["dep:rustfs-gateway", "dep:rustfs-gateway-adapter"]
|
||||
pyroscope = ["rustfs-obs/pyroscope", "dep:pyroscope", "dep:object", "dep:symbolic-demangle"]
|
||||
# Tokio runtime telemetry. Requires `--cfg tokio_unstable`; use `make build-profiling`.
|
||||
dial9 = ["rustfs-obs/dial9"]
|
||||
@@ -268,8 +255,6 @@ rustfs-notify = { workspace = true }
|
||||
rustfs-obs = { workspace = true }
|
||||
rustfs-policy = { workspace = true }
|
||||
rustfs-protocols = { workspace = true }
|
||||
rustfs-gateway = { workspace = true, optional = true }
|
||||
rustfs-gateway-adapter = { workspace = true, optional = true }
|
||||
rustfs-gateway-types = { workspace = true, features = ["compat-s3s-rustfs"] }
|
||||
rustfs-protos = { workspace = true }
|
||||
rustfs-rio = { workspace = true }
|
||||
|
||||
@@ -165,8 +165,7 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
let signing_secret = req
|
||||
.credentials()
|
||||
.ok_or_else(|| s3::error(S3ErrorCode::AccessDenied, "authentication required"))?
|
||||
.require_secret_key()
|
||||
.map_err(|err| s3::error(S3ErrorCode::InternalError, err.to_string()))?
|
||||
.secret_key()
|
||||
.to_string();
|
||||
let caller = CallerIdentity::resolve_envelope(req.envelope()).await?;
|
||||
let audit = AccountAuditContext::from_envelope(req.envelope());
|
||||
|
||||
@@ -63,10 +63,8 @@ impl Operation for RecoverOrphanedBucketHandler {
|
||||
authorize_admin_envelope(req.envelope(), vec![Action::AdminAction(AdminAction::RecoverOrphanedBucketAction)]).await?;
|
||||
let secret_key = req
|
||||
.credentials()
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "authentication required"))?
|
||||
.require_secret_key()
|
||||
.map(|secret| Zeroizing::new(secret.to_owned()))
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, err.to_string()))?;
|
||||
.map(|credentials| Zeroizing::new(credentials.secret_key().to_owned()))
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::InvalidRequest, "authentication required"))?;
|
||||
let bucket = params
|
||||
.get("bucket")
|
||||
.filter(|bucket| !bucket.is_empty())
|
||||
|
||||
@@ -15,12 +15,11 @@
|
||||
use crate::admin::operation::{AdminRequest, AdminResponse, Operation};
|
||||
use crate::admin::router::{ADMIN_OBJECT_ZIP_DOWNLOADS_PATH, AdminOperation, S3Router};
|
||||
use crate::admin::runtime_sources::{ServerContextSlot, current_action_credentials, current_object_store_handle, current_region};
|
||||
use crate::admin::storage_api::access::{ReqInfo, authorize_owned_request, authorize_request};
|
||||
use crate::admin::storage_api::access::{ReqInfo, authorize_request};
|
||||
use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::admin::storage_api::contract::list::ListOperations as _;
|
||||
use crate::admin::storage_api::contract::object::{ObjectIO as _, ObjectOperations as _};
|
||||
use crate::admin::storage_api::object::StorageObjectOptions as ObjectOptions;
|
||||
use crate::app::{RequestEnvelope, RequestHead};
|
||||
use crate::auth::{check_key_valid, get_session_token};
|
||||
use crate::error::ApiError;
|
||||
use crate::license::license_check;
|
||||
@@ -496,32 +495,25 @@ async fn authorize_object_zip_download_s3_action(
|
||||
req_info.version_id = None;
|
||||
}
|
||||
|
||||
let mut extensions = http::Extensions::new();
|
||||
extensions.insert(req.remote_addr().map(RemoteAddr));
|
||||
if let Some(client_info) = req.client_info() {
|
||||
extensions.insert(client_info.clone());
|
||||
}
|
||||
if let Some(server_context) = req.envelope().server_context() {
|
||||
extensions.insert(server_context.clone());
|
||||
}
|
||||
if let Some(req_info) = req.req_info() {
|
||||
extensions.insert(req_info.clone());
|
||||
}
|
||||
let mut authorization = RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method: req.method().clone(),
|
||||
uri: req.uri().clone(),
|
||||
headers: req.headers().clone(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
extensions,
|
||||
let mut authorization = s3_authorization_request(
|
||||
req.method().clone(),
|
||||
req.uri().clone(),
|
||||
&authenticated_zip_download_auth_context(req),
|
||||
req.envelope().server_context().cloned(),
|
||||
req.req_info().cloned(),
|
||||
);
|
||||
authorize_owned_request(&mut authorization, Action::S3Action(action), None).await
|
||||
authorize_request(&mut authorization, Action::S3Action(action))
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Builds the legacy authorization request used when a download token is redeemed.
|
||||
/// The S3 gate (`storage::access::authorize_request`) still evaluates the s3s
|
||||
/// request shape, so the facts it reads are copied into one for the duration
|
||||
/// of the call: the authenticated `ReqInfo`, the peer and proxy-reported client
|
||||
/// addresses, the headers and query the policy conditions come from, and the
|
||||
/// server slot that selects the store. The gate reads neither the presented
|
||||
/// credentials nor the region or service, so those stay unset, and nothing it
|
||||
/// attaches to the request is read afterwards.
|
||||
fn s3_authorization_request(
|
||||
method: Method,
|
||||
uri: Uri,
|
||||
@@ -951,103 +943,6 @@ mod tests {
|
||||
use super::*;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
|
||||
#[tokio::test]
|
||||
async fn zip_scope_authorizes_real_iam_and_restores_prefix_query() {
|
||||
use crate::admin::operation::test_support::TestRequest;
|
||||
use crate::admin::runtime_sources::{AppContext, IamInterface, KmsInterface};
|
||||
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use rustfs_iam::store::{Store as _, UserType};
|
||||
use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq};
|
||||
use rustfs_policy::policy::Policy;
|
||||
|
||||
struct ReadyIam(Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>>);
|
||||
impl IamInterface for ReadyIam {
|
||||
fn handle(&self) -> Arc<rustfs_iam::sys::IamSys<rustfs_iam::store::object::ObjectStore>> {
|
||||
Arc::clone(&self.0)
|
||||
}
|
||||
fn is_ready(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
struct TestKms;
|
||||
impl KmsInterface for TestKms {
|
||||
fn handle(&self) -> Arc<rustfs_kms::KmsServiceManager> {
|
||||
Arc::new(rustfs_kms::KmsServiceManager::new())
|
||||
}
|
||||
}
|
||||
|
||||
let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
store
|
||||
.make_bucket("photos", &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
rustfs_iam::store::object::ObjectStore::new(Arc::clone(&store))
|
||||
.save_iam_config(
|
||||
serde_json::json!({"version": 1}),
|
||||
format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX),
|
||||
)
|
||||
.await
|
||||
.expect("seed IAM format");
|
||||
let iam = rustfs_iam::build_iam_sys(Arc::clone(&store)).await.expect("initialize IAM");
|
||||
let access_key = "zip-scope-user";
|
||||
iam.create_user(
|
||||
access_key,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: "zip-scope-secret-key".to_owned(),
|
||||
policy: None,
|
||||
status: AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create user");
|
||||
iam.set_policy(
|
||||
"zip-scope-policy",
|
||||
Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListBucket","Resource":"arn:aws:s3:::photos","Condition":{"StringEquals":{"s3:prefix":"allowed/"}}},{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::photos/allowed/*"}]}"#)
|
||||
.expect("parse policy"),
|
||||
)
|
||||
.await
|
||||
.expect("store policy");
|
||||
iam.policy_db_set(access_key, UserType::Reg, false, "zip-scope-policy")
|
||||
.await
|
||||
.expect("attach policy");
|
||||
let credentials = iam.get_user(access_key).await.expect("load user").credentials;
|
||||
let context = Arc::new(AppContext::new(store, Arc::new(ReadyIam(iam)), Arc::new(TestKms)));
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let original_uri = "/rustfs/admin/v3/object-zip-downloads?prefix=old&marker=1";
|
||||
let mut request = TestRequest::new(Method::POST, original_uri)
|
||||
.extension(slot)
|
||||
.extension(ReqInfo {
|
||||
cred: Some(credentials),
|
||||
..Default::default()
|
||||
})
|
||||
.build();
|
||||
|
||||
let mut selection = valid_request();
|
||||
selection.prefix = None;
|
||||
selection.prefixes = vec!["allowed/".to_owned()];
|
||||
selection.objects = vec!["allowed/file".to_owned()];
|
||||
authorize_object_zip_download_scope(&mut request, &selection)
|
||||
.await
|
||||
.expect("allowed prefix and object");
|
||||
assert_eq!(request.uri().to_string(), original_uri);
|
||||
|
||||
selection.prefixes = vec!["denied/".to_owned()];
|
||||
let err = authorize_object_zip_download_scope(&mut request, &selection)
|
||||
.await
|
||||
.expect_err("another prefix is denied");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
assert_eq!(request.uri().to_string(), original_uri);
|
||||
|
||||
selection.prefixes = vec!["allowed/".to_owned()];
|
||||
selection.objects = vec!["denied/file".to_owned()];
|
||||
let err = authorize_object_zip_download_scope(&mut request, &selection)
|
||||
.await
|
||||
.expect_err("another object is denied");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
assert_eq!(request.uri().to_string(), original_uri);
|
||||
}
|
||||
|
||||
fn valid_request() -> CreateObjectZipDownloadRequest {
|
||||
CreateObjectZipDownloadRequest {
|
||||
bucket: "photos".to_string(),
|
||||
|
||||
@@ -343,7 +343,7 @@ impl Operation for AddServiceAccount {
|
||||
let (cred, owner) =
|
||||
check_key_valid(get_session_token(req.uri(), req.headers()).unwrap_or_default(), req_cred.access_key()).await?;
|
||||
|
||||
if !req_cred.secret_key_matches(&cred.secret_key) {
|
||||
if !constant_time_eq(&cred.secret_key, req_cred.secret_key()) {
|
||||
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "access denied"));
|
||||
}
|
||||
|
||||
|
||||
@@ -222,7 +222,7 @@ async fn handle_assume_role(
|
||||
|
||||
// The signature was verified before the request body was consumed. A fresh
|
||||
// IAM lookup must not silently replace the secret that authenticated it.
|
||||
if !user.secret_key_matches(&cred.secret_key) {
|
||||
if !crate::auth::constant_time_eq(&cred.secret_key, user.secret_key()) {
|
||||
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use crate::admin::operation::{AdminRequest, AdminResponse};
|
||||
use crate::admin::runtime_sources;
|
||||
use crate::admin::runtime_sources::default_admin_usecase;
|
||||
use crate::admin::storage_api::access::{ReqInfo, authorize_owned_internal_object_request};
|
||||
use crate::admin::storage_api::access::{ReqInfo, authorize_internal_object_request};
|
||||
use crate::admin::storage_api::bucket::metadata::table_catalog_path_hash;
|
||||
use crate::admin::storage_api::runtime::ECStore;
|
||||
use crate::admin::utils::owned_empty_response;
|
||||
@@ -1088,8 +1088,12 @@ fn build_sensitive_json_response<T: Serialize>(status: StatusCode, body: &T) ->
|
||||
Ok(AdminResponse::with_headers(status, body, headers))
|
||||
}
|
||||
|
||||
// These handlers still use the legacy router while their request bodies migrate.
|
||||
// TODO(T1.12-codemod): move them to the owned operation boundary.
|
||||
// The commit-path handlers (`TableCommitObjectBackend::for_request`) authorize
|
||||
// exact object keys through `storage::access::authorize_internal_object_request`,
|
||||
// which still takes the s3s request. Until that path has an owned form they keep
|
||||
// the s3s request and read every other request fact through this owned view.
|
||||
// TODO(T1.12-codemod): remove once the generic object authorization path moves
|
||||
// to the request envelope.
|
||||
type LegacyRequest = s3s::S3Request<s3s::Body>;
|
||||
type LegacyResponse = s3s::S3Response<(StatusCode, s3s::Body)>;
|
||||
|
||||
@@ -1314,14 +1318,15 @@ fn install_table_catalog_s3_request_info(req: &mut LegacyRequest, principal: &Ta
|
||||
}
|
||||
|
||||
async fn authorize_table_catalog_s3_actions(
|
||||
req: &mut AdminRequest,
|
||||
req: &mut LegacyRequest,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
actions: &[S3Action],
|
||||
) -> S3Result<()> {
|
||||
let original = {
|
||||
let req_info = req
|
||||
.req_info_mut()
|
||||
.extensions
|
||||
.get_mut::<ReqInfo>()
|
||||
.ok_or_else(|| s3_error!(AccessDenied, "authentication required"))?;
|
||||
(
|
||||
req_info.bucket.replace(bucket.to_string()),
|
||||
@@ -1332,15 +1337,15 @@ async fn authorize_table_catalog_s3_actions(
|
||||
|
||||
let mut result = Ok(());
|
||||
for action in actions {
|
||||
if let Err(err) = authorize_owned_internal_object_request(req.envelope_mut(), Action::S3Action(*action)).await {
|
||||
if let Err(err) = authorize_internal_object_request(req, Action::S3Action(*action)).await {
|
||||
result = Err(err);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if let Some(req_info) = req.req_info_mut() {
|
||||
if let Some(req_info) = req.extensions.get_mut::<ReqInfo>() {
|
||||
(req_info.bucket, req_info.object, req_info.version_id) = original;
|
||||
}
|
||||
result
|
||||
result.map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn table_catalog_request_principal(req: &AdminRequest) -> S3Result<TableCatalogRequestPrincipal> {
|
||||
@@ -1378,7 +1383,7 @@ async fn table_catalog_request_principal(req: &AdminRequest) -> S3Result<TableCa
|
||||
|
||||
#[derive(Clone)]
|
||||
enum TableCommitObjectAuthorization {
|
||||
Request(Arc<tokio::sync::Mutex<AdminRequest>>),
|
||||
Request(Arc<tokio::sync::Mutex<LegacyRequest>>),
|
||||
Preauthorized,
|
||||
#[cfg(test)]
|
||||
Test {
|
||||
@@ -1454,7 +1459,7 @@ where
|
||||
fn for_request(backend: B, req: LegacyRequest) -> Self {
|
||||
Self::new(
|
||||
backend,
|
||||
TableCommitObjectAuthorization::Request(Arc::new(tokio::sync::Mutex::new(legacy_request_view(&req)))),
|
||||
TableCommitObjectAuthorization::Request(Arc::new(tokio::sync::Mutex::new(req))),
|
||||
rustfs_utils::get_env_bool(crate::table_catalog::ENV_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED, false),
|
||||
)
|
||||
}
|
||||
@@ -2088,7 +2093,7 @@ where
|
||||
let result = match &self.authorization {
|
||||
TableCommitObjectAuthorization::Request(req) => {
|
||||
let req = req.lock().await;
|
||||
authorize_table_warehouse_location_request(&req, table_bucket, location).await
|
||||
authorize_table_warehouse_location_request(&legacy_request_view(&req), table_bucket, location).await
|
||||
}
|
||||
TableCommitObjectAuthorization::Preauthorized => Ok(()),
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -77,7 +77,8 @@ fn table_catalog_handler_request(context: Arc<AppContext>, access_key: &str, sec
|
||||
load_table_request_builder(context, access_key, secret_key).build()
|
||||
}
|
||||
|
||||
/// Builds the legacy request accepted by the commit backend, which derives an owned view.
|
||||
/// The s3s form of [`table_catalog_handler_request`] for the commit backend,
|
||||
/// which still holds the legacy request (see `legacy_request_view`).
|
||||
// TODO(T1.12-codemod): remove with `TableCommitObjectBackend::for_request`'s s3s parameter.
|
||||
fn legacy_table_catalog_handler_request(
|
||||
context: Arc<AppContext>,
|
||||
@@ -1294,10 +1295,6 @@ async fn table_storage_credentials_are_rejected_by_catalog_authentication() {
|
||||
#[tokio::test]
|
||||
async fn warehouse_location_claim_checks_request_identity_and_relocation() {
|
||||
let (_temp_dir, _disk_paths, store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
store
|
||||
.make_bucket("warehouse", &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create warehouse bucket");
|
||||
rustfs_iam::store::object::ObjectStore::new(store.clone())
|
||||
.save_iam_config(
|
||||
serde_json::json!({"version": 1}),
|
||||
@@ -1335,18 +1332,6 @@ async fn warehouse_location_claim_checks_request_identity_and_relocation() {
|
||||
iam.policy_db_set(access_key, UserType::Reg, false, "location-creator")
|
||||
.await
|
||||
.expect("attach location policy");
|
||||
iam.set_policy(
|
||||
"owned-object-policy",
|
||||
Policy::parse_config(
|
||||
br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::warehouse/owned/*"},{"Effect":"Allow","Action":"s3:PutObject","Resource":"arn:aws:s3:::warehouse/owned/writable"}]}"#,
|
||||
)
|
||||
.expect("parse object policy"),
|
||||
)
|
||||
.await
|
||||
.expect("store object policy");
|
||||
iam.policy_db_set(access_key, UserType::Reg, false, "location-creator,owned-object-policy")
|
||||
.await
|
||||
.expect("attach object policy");
|
||||
let context = Arc::new(AppContext::new(store, Arc::new(RequestIam { handle: iam }), Arc::new(RequestKms)));
|
||||
assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
|
||||
access_key: "location-root".to_string(),
|
||||
@@ -1354,125 +1339,6 @@ async fn warehouse_location_claim_checks_request_identity_and_relocation() {
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
}));
|
||||
// Exercise the actual owned internal authorization path, including its temporary target.
|
||||
let mut legacy = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
let principal = table_catalog_request_principal(&legacy_request_view(&legacy))
|
||||
.await
|
||||
.expect("resolve principal");
|
||||
install_table_catalog_s3_request_info(&mut legacy, &principal).expect("install authorization context");
|
||||
let info = legacy.extensions.get_mut::<ReqInfo>().expect("request info");
|
||||
info.bucket = Some("original-bucket".to_owned());
|
||||
info.object = Some("original-object".to_owned());
|
||||
info.version_id = Some("original-version".to_owned());
|
||||
let mut owned = legacy_request_view(&legacy);
|
||||
for (object, allowed) in [("owned/allowed", true), ("shared/denied", false)] {
|
||||
let result = authorize_table_catalog_s3_actions(&mut owned, "warehouse", object, &[S3Action::GetObjectAction]).await;
|
||||
if allowed {
|
||||
result.expect("the exact owned object should be allowed");
|
||||
} else {
|
||||
assert_eq!(result.expect_err("the other object should be denied").code(), &S3ErrorCode::AccessDenied);
|
||||
}
|
||||
let info = owned.envelope().req_info().expect("request info remains installed");
|
||||
assert_eq!(info.bucket.as_deref(), Some("original-bucket"));
|
||||
assert_eq!(info.object.as_deref(), Some("original-object"));
|
||||
assert_eq!(info.version_id.as_deref(), Some("original-version"));
|
||||
}
|
||||
// The exact-key check selects internal-object mode: a request header the object
|
||||
// write does not apply (a malformed x-amz-tagging) must not reach the decision,
|
||||
// while the ordinary owned entry point still evaluates it and rejects it.
|
||||
let mut tagged = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
install_table_catalog_s3_request_info(&mut tagged, &principal).expect("install authorization context");
|
||||
tagged
|
||||
.headers
|
||||
.insert("x-amz-tagging", http::HeaderValue::from_static("classification=%ZZ"));
|
||||
let mut tagged = legacy_request_view(&tagged);
|
||||
authorize_table_catalog_s3_actions(&mut tagged, "warehouse", "owned/writable", &[S3Action::PutObjectAction])
|
||||
.await
|
||||
.expect("internal-object mode ignores request options the write does not apply");
|
||||
let info = tagged.req_info_mut().expect("request info");
|
||||
info.bucket = Some("warehouse".to_owned());
|
||||
info.object = Some("owned/writable".to_owned());
|
||||
crate::app::authorize_owned_request(
|
||||
tagged.envelope_mut(),
|
||||
rustfs_policy::policy::action::Action::S3Action(S3Action::PutObjectAction),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("ordinary mode evaluates the malformed tagging header");
|
||||
let objects = TestTableCatalogObjectBackend::default();
|
||||
objects
|
||||
.seed_object("warehouse", "owned/read-only", b"readable".to_vec())
|
||||
.await;
|
||||
objects.seed_object("warehouse", "shared/denied", b"unchanged".to_vec()).await;
|
||||
let backend = || {
|
||||
let mut request = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
install_table_catalog_s3_request_info(&mut request, &principal).expect("install authorization context");
|
||||
TableCommitObjectBackend::for_request(objects.clone(), request)
|
||||
};
|
||||
let allowed = backend();
|
||||
let object = allowed
|
||||
.read_object("warehouse", "owned/read-only")
|
||||
.await
|
||||
.expect("authorized read")
|
||||
.expect("seeded object");
|
||||
assert_eq!(object.data, b"readable");
|
||||
allowed
|
||||
.put_object(
|
||||
"warehouse",
|
||||
"owned/writable",
|
||||
b"written".to_vec(),
|
||||
crate::table_catalog::TableCatalogPutPrecondition::Any,
|
||||
)
|
||||
.await
|
||||
.expect("authorized write");
|
||||
assert_eq!(
|
||||
objects
|
||||
.read_object("warehouse", "owned/writable")
|
||||
.await
|
||||
.expect("read stored object")
|
||||
.expect("written object")
|
||||
.data,
|
||||
b"written"
|
||||
);
|
||||
for (object, write) in [("shared/denied", false), ("owned/read-only", true), ("shared/denied", true)] {
|
||||
let denied = backend();
|
||||
let result = if write {
|
||||
denied
|
||||
.put_object(
|
||||
"warehouse",
|
||||
object,
|
||||
b"modified".to_vec(),
|
||||
crate::table_catalog::TableCatalogPutPrecondition::Any,
|
||||
)
|
||||
.await
|
||||
.map(|_| ())
|
||||
} else {
|
||||
denied.read_object("warehouse", object).await.map(|_| ())
|
||||
};
|
||||
assert!(result.is_err(), "unauthorized {object} operation must fail");
|
||||
assert_eq!(
|
||||
denied.finish(Ok(())).await.expect_err("retain authorization error").code(),
|
||||
&S3ErrorCode::AccessDenied
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
objects
|
||||
.read_object("warehouse", "owned/read-only")
|
||||
.await
|
||||
.expect("read object")
|
||||
.expect("seeded object")
|
||||
.data,
|
||||
b"readable"
|
||||
);
|
||||
assert_eq!(
|
||||
objects
|
||||
.read_object("warehouse", "shared/denied")
|
||||
.await
|
||||
.expect("read object")
|
||||
.expect("seeded object")
|
||||
.data,
|
||||
b"unchanged"
|
||||
);
|
||||
let request = || table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
for location in ["s3://warehouse/owned", "s3://warehouse/owned/", "s3://warehouse/owned/child"] {
|
||||
authorize_table_warehouse_location_request(&request(), "warehouse", location)
|
||||
|
||||
@@ -243,10 +243,7 @@ impl Operation for AddUser {
|
||||
let (cred, owner) =
|
||||
check_key_valid(get_session_token(req.uri(), req.headers()).unwrap_or_default(), input_cred.access_key()).await?;
|
||||
// The caller's current secret outlives the body read below.
|
||||
let input_secret = input_cred
|
||||
.require_secret_key()
|
||||
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, err.to_string()))?
|
||||
.to_owned();
|
||||
let input_secret = input_cred.secret_key().to_owned();
|
||||
|
||||
let ak = query.access_key.as_deref().unwrap_or_default();
|
||||
|
||||
|
||||
@@ -210,10 +210,11 @@ pub(crate) mod test_support {
|
||||
/// token is read from the request like the s3s edge does, so a test
|
||||
/// that sets `x-amz-security-token` keeps working unchanged.
|
||||
pub(crate) fn credentials(mut self, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Self {
|
||||
self.credentials = Some(
|
||||
RequestCredentials::new(access_key.to_owned(), secret_key.to_owned(), session_token.map(str::to_owned))
|
||||
.expect("test credentials carry a non-empty secret"),
|
||||
);
|
||||
self.credentials = Some(RequestCredentials::new(
|
||||
access_key.to_owned(),
|
||||
secret_key.to_owned(),
|
||||
session_token.map(str::to_owned),
|
||||
));
|
||||
self
|
||||
}
|
||||
|
||||
@@ -245,7 +246,11 @@ pub(crate) mod test_support {
|
||||
pub(crate) fn build(self) -> AdminRequest {
|
||||
let credentials = self.credentials.map(|credentials| match credentials.session_token() {
|
||||
Some(_) => credentials,
|
||||
None => credentials.with_session_token(get_session_token(&self.uri, &self.headers).map(str::to_owned)),
|
||||
None => RequestCredentials::new(
|
||||
credentials.access_key().to_owned(),
|
||||
credentials.secret_key().to_owned(),
|
||||
get_session_token(&self.uri, &self.headers).map(str::to_owned),
|
||||
),
|
||||
});
|
||||
let head = RequestHead {
|
||||
method: self.method,
|
||||
@@ -293,7 +298,7 @@ mod tests {
|
||||
.build();
|
||||
let credentials = req.credentials().expect("credentials were supplied");
|
||||
assert_eq!(credentials.access_key(), "AKIATEST");
|
||||
assert_eq!(credentials.secret_key(), Some("secret"));
|
||||
assert_eq!(credentials.secret_key(), "secret");
|
||||
assert_eq!(credentials.session_token(), Some("from-header"));
|
||||
|
||||
let explicit = TestRequest::new(Method::GET, "/rustfs/admin/v3/info")
|
||||
|
||||
@@ -3882,7 +3882,7 @@ mod tests {
|
||||
.credentials()
|
||||
.ok_or_else(|| OwnedS3Error::with_message(OwnedS3ErrorCode::AccessDenied, "owned admin identity is required"))?;
|
||||
let identity_matches = credentials.access_key() == "AKIAOWNED"
|
||||
&& credentials.secret_key_matches("owned-secret")
|
||||
&& constant_time_eq(credentials.secret_key(), "owned-secret")
|
||||
&& credentials
|
||||
.session_token()
|
||||
.is_some_and(|token| constant_time_eq(token, "owned-session"));
|
||||
|
||||
@@ -953,8 +953,9 @@ pub(crate) mod data_usage {
|
||||
}
|
||||
|
||||
pub(crate) mod access {
|
||||
pub(crate) use crate::app::{authorize_owned_internal_object_request, authorize_owned_request};
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{ReqInfo, authorize_request};
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{
|
||||
ReqInfo, authorize_internal_object_request, authorize_request,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::request_context_consumer::{RequestContext, spawn_traced};
|
||||
}
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
|
||||
#[cfg(test)]
|
||||
use super::storage_api::bucket_usecase::access::ReqInfo;
|
||||
use super::storage_api::bucket_usecase::access::{
|
||||
ListBucketsAuthzRequest, TableDataPlaneListAccess, authorize_request, bucket_config_mutation_incarnation,
|
||||
log_list_buckets_iam_implicit_deny, prepare_list_buckets_iam_authorization, prepare_odm_read_generation, req_info_ref,
|
||||
TableDataPlaneListAccess, authorize_request, bucket_config_mutation_incarnation, log_list_buckets_iam_implicit_deny,
|
||||
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, req_info_ref,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::storage_api::bucket_usecase::bucket::target::BucketTarget;
|
||||
@@ -47,7 +47,7 @@ use super::storage_api::bucket_usecase::bucket::{
|
||||
versioning_sys::BucketVersioningSys,
|
||||
};
|
||||
use super::storage_api::bucket_usecase::contract::bucket::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
|
||||
BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
|
||||
};
|
||||
use super::storage_api::bucket_usecase::contract::list::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as _,
|
||||
@@ -56,9 +56,9 @@ use super::storage_api::bucket_usecase::error::StorageError;
|
||||
use super::storage_api::bucket_usecase::helper::{OperationHelper, spawn_background_with_context};
|
||||
use super::storage_api::bucket_usecase::object_utils::to_s3s_xml_etag;
|
||||
use super::storage_api::bucket_usecase::s3_api::bucket::{
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_object_versions_output, build_list_objects_output,
|
||||
build_list_objects_v2_output, encode_list_objects_v2_continuation_token, parse_list_object_versions_params,
|
||||
parse_list_objects_v2_params, rustfs_owner,
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
|
||||
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
|
||||
parse_list_object_versions_params, parse_list_objects_v2_params, rustfs_owner,
|
||||
};
|
||||
use super::storage_api::bucket_usecase::{
|
||||
get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
@@ -66,7 +66,6 @@ use super::storage_api::bucket_usecase::{
|
||||
};
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
use super::storage_api::diagnostic;
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::bucket_list_through as list_through;
|
||||
use crate::app::object::shared::mark_on_demand_migration_list_local_only;
|
||||
use crate::app::object_data_cache::invalidate_object_data_cache_bucket_after_delete;
|
||||
@@ -88,14 +87,12 @@ use http::StatusCode;
|
||||
use metrics::counter;
|
||||
use rustfs_config::RUSTFS_REGION;
|
||||
use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration, PersistedTagging};
|
||||
use rustfs_gateway_types::dto as gateway_dto;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedBucketEncryptionConfiguration, PersistedNotificationConfiguration,
|
||||
PersistedObjectLockConfiguration as ObjectLockConfiguration, PersistedPublicAccessBlockConfiguration,
|
||||
PersistedVersioningConfiguration,
|
||||
};
|
||||
use rustfs_gateway_types::persistence::{PersistedLifecycleConfiguration, PersistedReplicationConfiguration};
|
||||
use rustfs_gateway_types::{BucketName, Timestamp as GatewayTimestamp};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
|
||||
use rustfs_policy::policy::{
|
||||
@@ -119,15 +116,15 @@ use s3s::dto::{
|
||||
GetBucketCorsInput, GetBucketEncryptionInput, GetBucketLifecycleConfigurationInput, GetBucketLocationInput,
|
||||
GetBucketLocationOutput, GetBucketNotificationConfigurationInput, GetBucketPolicyInput, GetBucketPolicyOutput,
|
||||
GetBucketPolicyStatusInput, GetBucketPolicyStatusOutput, GetBucketReplicationInput, GetBucketTaggingInput,
|
||||
GetBucketVersioningInput, GetPublicAccessBlockInput, HeadBucketInput, HeadBucketOutput, LifecycleRule,
|
||||
ListObjectVersionsInput, ListObjectVersionsOutput, ListObjectsInput, ListObjectsOutput, ListObjectsV2Input,
|
||||
ListObjectsV2Output, MetadataEntry, NotificationConfiguration, NotificationConfigurationFilter, Object, ObjectStorageClass,
|
||||
ObjectVersion, ObjectVersionStorageClass, PolicyStatus, PutBucketCorsInput, PutBucketCorsOutput, PutBucketEncryptionInput,
|
||||
PutBucketEncryptionOutput, PutBucketLifecycleConfigurationInput, PutBucketLifecycleConfigurationOutput,
|
||||
PutBucketNotificationConfigurationInput, PutBucketNotificationConfigurationOutput, PutBucketPolicyInput,
|
||||
PutBucketPolicyOutput, PutBucketReplicationInput, PutBucketReplicationOutput, PutBucketTaggingInput, PutBucketTaggingOutput,
|
||||
PutBucketVersioningInput, PutBucketVersioningOutput, PutPublicAccessBlockInput, PutPublicAccessBlockOutput,
|
||||
ServerSideEncryption, Timestamp, UserMetadata,
|
||||
GetBucketVersioningInput, GetPublicAccessBlockInput, HeadBucketInput, HeadBucketOutput, LifecycleRule, ListBucketsInput,
|
||||
ListBucketsOutput, ListObjectVersionsInput, ListObjectVersionsOutput, ListObjectsInput, ListObjectsOutput,
|
||||
ListObjectsV2Input, ListObjectsV2Output, MetadataEntry, NotificationConfiguration, NotificationConfigurationFilter, Object,
|
||||
ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, PolicyStatus, PutBucketCorsInput, PutBucketCorsOutput,
|
||||
PutBucketEncryptionInput, PutBucketEncryptionOutput, PutBucketLifecycleConfigurationInput,
|
||||
PutBucketLifecycleConfigurationOutput, PutBucketNotificationConfigurationInput, PutBucketNotificationConfigurationOutput,
|
||||
PutBucketPolicyInput, PutBucketPolicyOutput, PutBucketReplicationInput, PutBucketReplicationOutput, PutBucketTaggingInput,
|
||||
PutBucketTaggingOutput, PutBucketVersioningInput, PutBucketVersioningOutput, PutPublicAccessBlockInput,
|
||||
PutPublicAccessBlockOutput, ServerSideEncryption, Timestamp, UserMetadata,
|
||||
};
|
||||
use s3s::region::Region;
|
||||
use s3s::xml;
|
||||
@@ -1406,40 +1403,6 @@ fn lifecycle_has_abort_multipart_rules(config: &PersistedLifecycleConfiguration)
|
||||
.any(|rule| rule.status == "Enabled" && rule.abort_incomplete_multipart_upload.is_some())
|
||||
}
|
||||
|
||||
/// The ListBuckets answer for `bucket_infos`, in the gateway's types.
|
||||
///
|
||||
/// A stored name the S3 bucket grammar refuses cannot be written into the answer and is left out.
|
||||
/// The storage inventory holds such names only for RustFS-internal buckets (the SFTP host-key
|
||||
/// store); `rustfs/src/protocols/client.rs` applies the same rule to its listings.
|
||||
fn list_buckets_output(bucket_infos: &[BucketInfo]) -> gateway_dto::ListBucketsOutput {
|
||||
let buckets = bucket_infos
|
||||
.iter()
|
||||
.filter_map(|bucket_info| {
|
||||
let name = BucketName::new(bucket_info.name.clone()).ok()?;
|
||||
let created = bucket_info.created.unwrap_or(time::OffsetDateTime::UNIX_EPOCH);
|
||||
let seconds = created.unix_timestamp();
|
||||
let creation_date =
|
||||
GatewayTimestamp::from_secs_nanos(seconds, created.nanosecond()).unwrap_or(GatewayTimestamp::from_secs(seconds));
|
||||
Some(gateway_dto::Bucket {
|
||||
name,
|
||||
creation_date,
|
||||
..Default::default()
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let owner = rustfs_owner();
|
||||
|
||||
gateway_dto::ListBucketsOutput {
|
||||
buckets,
|
||||
owner: gateway_dto::Owner {
|
||||
id: owner.id,
|
||||
display_name: owner.display_name,
|
||||
},
|
||||
continuation_token: None,
|
||||
prefix: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct DefaultBucketUsecase {
|
||||
context: Option<Arc<AppContext>>,
|
||||
@@ -1776,61 +1739,21 @@ impl DefaultBucketUsecase {
|
||||
Ok(S3Response::new(GetBucketLocationOutput { location_constraint }))
|
||||
}
|
||||
|
||||
/// Lists the buckets the caller of `envelope` may see, for either edge.
|
||||
///
|
||||
/// The access phase must have installed the caller's `ReqInfo` in `envelope`. A caller allowed
|
||||
/// `s3:ListAllMyBuckets` sees every bucket; any other caller sees the buckets it may list or
|
||||
/// locate, and is refused when there is none. `_input` is accepted and not read: RustFS answers
|
||||
/// one unpaginated listing and ignores `prefix`, `max-buckets`, `continuation-token` and
|
||||
/// `bucket-region`, as it always has.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// `AccessDenied` for an anonymous caller and for one that may see no bucket; `InternalError`
|
||||
/// when the store or IAM is not initialized or the access phase did not run; otherwise the
|
||||
/// storage error of the listing.
|
||||
#[instrument(level = "debug", skip(self, envelope, _input))]
|
||||
pub async fn execute_list_buckets(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
_input: gateway_dto::ListBucketsInput,
|
||||
) -> rustfs_s3_types::S3Result<gateway_dto::ListBucketsOutput> {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_list_buckets(&self, req: S3Request<ListBucketsInput>) -> S3Result<S3Response<ListBucketsOutput>> {
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(rustfs_s3_types::S3Error::with_message(
|
||||
rustfs_s3_types::S3ErrorCode::InternalError,
|
||||
"Not init",
|
||||
));
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
};
|
||||
|
||||
if envelope
|
||||
.credentials()
|
||||
.is_none_or(|credentials| credentials.access_key().is_empty())
|
||||
{
|
||||
return Err(rustfs_s3_types::S3Error::with_message(
|
||||
rustfs_s3_types::S3ErrorCode::AccessDenied,
|
||||
"Access Denied",
|
||||
));
|
||||
if req.credentials.as_ref().is_none_or(|cred| cred.access_key.is_empty()) {
|
||||
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
|
||||
}
|
||||
|
||||
let Some(req_info) = envelope.req_info() else {
|
||||
return Err(rustfs_s3_types::S3Error::with_message(
|
||||
rustfs_s3_types::S3ErrorCode::InternalError,
|
||||
"ReqInfo not found in request envelope",
|
||||
));
|
||||
};
|
||||
let iam_authorization = prepare_list_buckets_iam_authorization(ListBucketsAuthzRequest {
|
||||
req_info,
|
||||
headers: envelope.headers(),
|
||||
raw_query: envelope.uri().query(),
|
||||
remote_addr: envelope.remote_addr(),
|
||||
client_info: envelope.client_info(),
|
||||
server_context: envelope.server_context(),
|
||||
})
|
||||
.await?;
|
||||
let iam_authorization = prepare_list_buckets_iam_authorization(&req).await?;
|
||||
let bucket_infos = if iam_authorization.is_allowed("", S3Action::ListAllMyBucketsAction).await {
|
||||
store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?
|
||||
} else {
|
||||
log_list_buckets_iam_implicit_deny(req_info);
|
||||
log_list_buckets_iam_implicit_deny(&req)?;
|
||||
let bucket_infos = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?;
|
||||
let mut visible_bucket_infos = Vec::new();
|
||||
for info in bucket_infos {
|
||||
@@ -1849,7 +1772,7 @@ impl DefaultBucketUsecase {
|
||||
visible_bucket_infos
|
||||
};
|
||||
|
||||
Ok(list_buckets_output(&bucket_infos))
|
||||
Ok(S3Response::new(build_list_buckets_output(&bucket_infos)))
|
||||
}
|
||||
|
||||
pub async fn execute_delete_bucket_encryption(
|
||||
@@ -5161,60 +5084,13 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_buckets_returns_internal_error_when_store_uninitialized() {
|
||||
let req = build_request((), Method::GET);
|
||||
let input = ListBucketsInput::builder().build().unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_list_buckets(RequestEnvelope::from_s3s(&req), gateway_dto::ListBucketsInput::default())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
fn bucket_info(name: &str, created: Option<time::OffsetDateTime>) -> BucketInfo {
|
||||
BucketInfo {
|
||||
name: name.to_owned(),
|
||||
created,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_buckets_output_carries_names_dates_and_the_owner() {
|
||||
let created = time::OffsetDateTime::from_unix_timestamp_nanos(1_700_000_000_123_000_000).expect("valid timestamp");
|
||||
|
||||
let output = list_buckets_output(&[bucket_info("alpha", Some(created)), bucket_info("beta", None)]);
|
||||
|
||||
let listed: Vec<(&str, i64, u32)> = output
|
||||
.buckets
|
||||
.iter()
|
||||
.map(|bucket| (bucket.name.as_str(), bucket.creation_date.secs(), bucket.creation_date.subsec_nanos()))
|
||||
.collect();
|
||||
assert_eq!(listed, [("alpha", 1_700_000_000, 123_000_000), ("beta", 0, 0)]);
|
||||
assert!(output.owner.id.is_some(), "the listing names its owner");
|
||||
assert_eq!(output.continuation_token, None);
|
||||
assert_eq!(output.prefix, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_buckets_output_leaves_out_names_the_bucket_grammar_refuses() {
|
||||
let output = list_buckets_output(&[
|
||||
bucket_info("sftp_host_keys", None),
|
||||
bucket_info("UPPER", None),
|
||||
bucket_info("ab", None),
|
||||
bucket_info("kept", None),
|
||||
]);
|
||||
|
||||
let names: Vec<&str> = output.buckets.iter().map(|bucket| bucket.name.as_str()).collect();
|
||||
assert_eq!(names, ["kept"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_buckets_output_of_no_bucket_is_an_empty_listing_with_an_owner() {
|
||||
let output = list_buckets_output(&[]);
|
||||
|
||||
assert!(output.buckets.is_empty());
|
||||
assert!(output.owner.id.is_some());
|
||||
let err = usecase.execute_list_buckets(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -26,7 +26,6 @@ mod startup;
|
||||
|
||||
pub(crate) use envelope::RequestHead;
|
||||
pub use envelope::{RequestCredentials, RequestEnvelope};
|
||||
pub(crate) use envelope::{authorize_owned_internal_object_request, authorize_owned_request};
|
||||
pub use global::*;
|
||||
pub use handles::*;
|
||||
pub use interfaces::*;
|
||||
|
||||
@@ -29,19 +29,13 @@ use super::ServerContextSlot;
|
||||
use crate::app::metadata_route::WebsiteRead;
|
||||
use crate::app::object::request_body::BodyReadControl;
|
||||
use crate::app::storage_api::context::{
|
||||
AuditRequestView, AuthorizationRequest, DenialAuditor, DenialScope, authorize_request_core,
|
||||
};
|
||||
use crate::app::storage_api::context::{
|
||||
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, MultipartStagingOperation,
|
||||
ObjectTagConditions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectPolicyHeaders,
|
||||
PostObjectRequestMarker, ReqInfo, RequestContext, RequestedCopySource, RequestedObjectLegalHold,
|
||||
RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags, StagedMultipartPartAuthorization,
|
||||
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, ObjectTagConditions,
|
||||
OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectRequestMarker, ReqInfo, RequestContext,
|
||||
TableDataPlaneListAccess, TableDataPlanePublicationGuards,
|
||||
};
|
||||
use crate::auth::{VerifiedPresignedRequest, VerifiedSigV4Request};
|
||||
use crate::shared_types::{NativeTlsTransport, RemoteAddr};
|
||||
use http::{Extensions, HeaderMap, Method, Uri};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::fmt;
|
||||
use std::net::SocketAddr;
|
||||
@@ -51,109 +45,42 @@ use zeroize::Zeroizing;
|
||||
/// The identity a request was authenticated with.
|
||||
///
|
||||
/// The secret key and the session token are bearer material: the type has no
|
||||
/// `PartialEq`, and its `Debug` output redacts both. An edge that keeps the
|
||||
/// caller's secret (it verified the signature itself) builds the value with
|
||||
/// [`Self::without_secret_key`]; the absence is then `None`, never a
|
||||
/// substitute string a reader could compare, seal or derive with.
|
||||
/// `PartialEq`, and its `Debug` output redacts both.
|
||||
#[derive(Clone)]
|
||||
pub struct RequestCredentials {
|
||||
access_key: String,
|
||||
secret_key: Option<Zeroizing<String>>,
|
||||
secret_key: Zeroizing<String>,
|
||||
session_token: Option<Zeroizing<String>>,
|
||||
}
|
||||
|
||||
/// An edge handed over an empty secret key. Absence is expressed with
|
||||
/// [`RequestCredentials::without_secret_key`], so an empty secret is refused
|
||||
/// rather than read as "absent" or used as a key.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct EmptySecretKey;
|
||||
|
||||
impl fmt::Display for EmptySecretKey {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("handed-over secret key is empty")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for EmptySecretKey {}
|
||||
|
||||
impl RequestCredentials {
|
||||
/// Credentials whose secret the edge handed over.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`EmptySecretKey`] when `secret_key` is empty.
|
||||
pub(crate) fn new(access_key: String, secret_key: String, session_token: Option<String>) -> Result<Self, EmptySecretKey> {
|
||||
if secret_key.is_empty() {
|
||||
return Err(EmptySecretKey);
|
||||
}
|
||||
Ok(Self {
|
||||
access_key,
|
||||
secret_key: Some(Zeroizing::new(secret_key)),
|
||||
session_token: session_token.map(Zeroizing::new),
|
||||
})
|
||||
}
|
||||
|
||||
/// Credentials whose secret the edge kept.
|
||||
pub(crate) fn without_secret_key(access_key: String, session_token: Option<String>) -> Self {
|
||||
pub(crate) fn new(access_key: String, secret_key: String, session_token: Option<String>) -> Self {
|
||||
Self {
|
||||
access_key,
|
||||
secret_key: None,
|
||||
secret_key: Zeroizing::new(secret_key),
|
||||
session_token: session_token.map(Zeroizing::new),
|
||||
}
|
||||
}
|
||||
|
||||
/// The same identity with `session_token` as its token.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_session_token(mut self, session_token: Option<String>) -> Self {
|
||||
self.session_token = session_token.map(Zeroizing::new);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn access_key(&self) -> &str {
|
||||
&self.access_key
|
||||
}
|
||||
|
||||
/// The caller's secret, or `None` when the edge did not hand it over.
|
||||
/// A reader must fail closed on `None`. Never log this value or compare it
|
||||
/// with `==`.
|
||||
pub fn secret_key(&self) -> Option<&str> {
|
||||
self.secret_key.as_deref().map(String::as_str)
|
||||
/// Never log this value or compare it with `==`.
|
||||
pub fn secret_key(&self) -> &str {
|
||||
&self.secret_key
|
||||
}
|
||||
|
||||
pub fn session_token(&self) -> Option<&str> {
|
||||
self.session_token.as_deref().map(String::as_str)
|
||||
}
|
||||
|
||||
/// The handed-over secret, or [`MissingSecretKey`] when the edge kept it.
|
||||
pub(crate) fn require_secret_key(&self) -> Result<&str, MissingSecretKey> {
|
||||
self.secret_key().ok_or(MissingSecretKey)
|
||||
}
|
||||
|
||||
/// Whether the handed-over secret equals `stored`, compared in constant
|
||||
/// time. An absent secret matches nothing, not even an empty `stored`.
|
||||
pub(crate) fn secret_key_matches(&self, stored: &str) -> bool {
|
||||
self.secret_key()
|
||||
.is_some_and(|secret| crate::auth::constant_time_eq(secret, stored))
|
||||
}
|
||||
}
|
||||
|
||||
/// A reader needed the caller's secret, but the edge did not hand it over.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct MissingSecretKey;
|
||||
|
||||
impl fmt::Display for MissingSecretKey {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("the caller's secret key was not handed over for this request")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for MissingSecretKey {}
|
||||
|
||||
impl fmt::Debug for RequestCredentials {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("RequestCredentials")
|
||||
.field("access_key", &self.access_key)
|
||||
.field("secret_key", &self.secret_key.as_ref().map(|_| "<redacted>"))
|
||||
.field("secret_key", &"<redacted>")
|
||||
.field("session_token", &self.session_token.as_ref().map(|_| "<redacted>"))
|
||||
.finish()
|
||||
}
|
||||
@@ -201,7 +128,6 @@ pub struct RequestEnvelope {
|
||||
verified_presigned: Option<VerifiedPresignedRequest>,
|
||||
website_read: Option<WebsiteRead>,
|
||||
post_object: Option<PostObjectRequestMarker>,
|
||||
post_object_policy_headers: Option<PostObjectPolicyHeaders>,
|
||||
// Attached by the authorization phase (`storage/access.rs`) for the use case.
|
||||
req_info: Option<ReqInfo>,
|
||||
bucket_generation: Option<BucketGenerationGuard>,
|
||||
@@ -212,13 +138,6 @@ pub struct RequestEnvelope {
|
||||
table_publication_guards: Option<TableDataPlanePublicationGuards>,
|
||||
table_list_access: Option<TableDataPlaneListAccess>,
|
||||
object_tag_conditions: Option<ObjectTagConditions>,
|
||||
multipart_staging_operation: Option<MultipartStagingOperation>,
|
||||
staged_multipart_part_authorization: Option<StagedMultipartPartAuthorization>,
|
||||
requested_object_lock_retention: Option<RequestedObjectLockRetention>,
|
||||
requested_object_lock_retain_until: Option<RequestedObjectLockRetainUntil>,
|
||||
requested_object_legal_hold: Option<RequestedObjectLegalHold>,
|
||||
requested_object_tags: Option<RequestedObjectTags>,
|
||||
requested_copy_source: Option<RequestedCopySource>,
|
||||
residual_extensions: Extensions,
|
||||
}
|
||||
|
||||
@@ -248,7 +167,6 @@ impl RequestEnvelope {
|
||||
verified_presigned: extensions.remove(),
|
||||
website_read: extensions.remove(),
|
||||
post_object: extensions.remove(),
|
||||
post_object_policy_headers: extensions.remove(),
|
||||
req_info: extensions.remove(),
|
||||
bucket_generation: extensions.remove(),
|
||||
pending_delete_bucket_generation: extensions.remove(),
|
||||
@@ -258,13 +176,6 @@ impl RequestEnvelope {
|
||||
table_publication_guards: extensions.remove(),
|
||||
table_list_access: extensions.remove(),
|
||||
object_tag_conditions: extensions.remove(),
|
||||
multipart_staging_operation: extensions.remove(),
|
||||
staged_multipart_part_authorization: extensions.remove(),
|
||||
requested_object_lock_retention: extensions.remove(),
|
||||
requested_object_lock_retain_until: extensions.remove(),
|
||||
requested_object_legal_hold: extensions.remove(),
|
||||
requested_object_tags: extensions.remove(),
|
||||
requested_copy_source: extensions.remove(),
|
||||
residual_extensions: extensions,
|
||||
}
|
||||
}
|
||||
@@ -401,10 +312,6 @@ impl RequestEnvelope {
|
||||
self.table_publication_guards.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn install_table_publication_guards(&mut self, guards: TableDataPlanePublicationGuards) {
|
||||
self.table_publication_guards = Some(guards);
|
||||
}
|
||||
|
||||
pub(crate) fn table_list_access(&self) -> Option<&TableDataPlaneListAccess> {
|
||||
self.table_list_access.as_ref()
|
||||
}
|
||||
@@ -412,337 +319,19 @@ impl RequestEnvelope {
|
||||
pub(crate) fn object_tag_conditions(&self) -> Option<&ObjectTagConditions> {
|
||||
self.object_tag_conditions.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn install_object_tag_conditions(&mut self, conditions: ObjectTagConditions) {
|
||||
self.object_tag_conditions = Some(conditions);
|
||||
}
|
||||
|
||||
pub(crate) fn multipart_staging_operation(&self) -> Option<&MultipartStagingOperation> {
|
||||
self.multipart_staging_operation.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn staged_multipart_part_authorization(&self) -> Option<&StagedMultipartPartAuthorization> {
|
||||
self.staged_multipart_part_authorization.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn requested_object_lock_retention(&self) -> Option<&RequestedObjectLockRetention> {
|
||||
self.requested_object_lock_retention.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn requested_object_lock_retain_until(&self) -> Option<&RequestedObjectLockRetainUntil> {
|
||||
self.requested_object_lock_retain_until.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn post_object_policy_headers(&self) -> Option<&PostObjectPolicyHeaders> {
|
||||
self.post_object_policy_headers.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn requested_object_legal_hold(&self) -> Option<&RequestedObjectLegalHold> {
|
||||
self.requested_object_legal_hold.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn requested_object_tags(&self) -> Option<&RequestedObjectTags> {
|
||||
self.requested_object_tags.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn requested_copy_source(&self) -> Option<&RequestedCopySource> {
|
||||
self.requested_copy_source.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
/// The authorization core in `storage` reads requests only through
|
||||
/// [`AuthorizationRequest`]; this impl hands it the envelope's named fields.
|
||||
impl AuthorizationRequest for RequestEnvelope {
|
||||
fn headers(&self) -> &HeaderMap {
|
||||
self.headers()
|
||||
}
|
||||
fn uri(&self) -> &http::Uri {
|
||||
self.uri()
|
||||
}
|
||||
fn remote_addr(&self) -> Option<std::net::SocketAddr> {
|
||||
self.remote_addr()
|
||||
}
|
||||
fn client_info(&self) -> Option<&ClientInfo> {
|
||||
self.client_info()
|
||||
}
|
||||
fn server_context(&self) -> Option<&Arc<ServerContextSlot>> {
|
||||
self.server_context()
|
||||
}
|
||||
fn req_info(&self) -> Option<&ReqInfo> {
|
||||
self.req_info()
|
||||
}
|
||||
fn internal_object_authorization(&self) -> bool {
|
||||
false
|
||||
}
|
||||
fn multipart_staging(&self) -> bool {
|
||||
self.multipart_staging_operation().is_some()
|
||||
}
|
||||
fn staged_multipart_part(&self) -> bool {
|
||||
self.staged_multipart_part_authorization().is_some()
|
||||
}
|
||||
fn requested_retention(&self) -> Option<&RequestedObjectLockRetention> {
|
||||
self.requested_object_lock_retention()
|
||||
}
|
||||
fn requested_retain_until(&self) -> Option<&RequestedObjectLockRetainUntil> {
|
||||
self.requested_object_lock_retain_until()
|
||||
}
|
||||
fn requested_legal_hold(&self) -> Option<&RequestedObjectLegalHold> {
|
||||
self.requested_object_legal_hold()
|
||||
}
|
||||
fn requested_object_tags(&self) -> Option<&RequestedObjectTags> {
|
||||
self.requested_object_tags()
|
||||
}
|
||||
fn requested_copy_source(&self) -> Option<&RequestedCopySource> {
|
||||
self.requested_copy_source()
|
||||
}
|
||||
fn policy_headers(&self) -> &HeaderMap {
|
||||
self.post_object_policy_headers().map_or(self.headers(), |headers| &headers.0)
|
||||
}
|
||||
fn object_tag_conditions(&self) -> Option<&ObjectTagConditions> {
|
||||
self.object_tag_conditions()
|
||||
}
|
||||
fn install_object_tag_conditions(&mut self, conditions: ObjectTagConditions) {
|
||||
self.install_object_tag_conditions(conditions);
|
||||
}
|
||||
fn publication_guards(&self) -> Option<&TableDataPlanePublicationGuards> {
|
||||
self.table_publication_guards()
|
||||
}
|
||||
fn install_publication_guards(&mut self, guards: TableDataPlanePublicationGuards) {
|
||||
self.install_table_publication_guards(guards);
|
||||
}
|
||||
fn audit_denial(&self, auditor: &DenialAuditor, action: &Action, reason: &'static str) {
|
||||
auditor.audit_owned(&self.audit_view(), action, reason, DenialScope::Request);
|
||||
}
|
||||
}
|
||||
|
||||
/// Authorizes an owned request through the same IAM, bucket and table policies as the legacy edge.
|
||||
///
|
||||
/// Like the legacy `authorize_request`, it first restores the signed `x-amz-meta-*`
|
||||
/// query parameters into headers, so policy conditions and duplicate rejection see
|
||||
/// the same request shape on both edges.
|
||||
///
|
||||
/// This is only the policy decision. It does **not** perform the composite checks
|
||||
/// the legacy `S3Access` hooks run around their decision, such as the
|
||||
/// `s3:BypassGovernanceRetention` authorization for a governance-bypass header or
|
||||
/// the copy-source read authorization; a caller that needs them runs them itself.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// `InvalidArgument` when the signed query repeats a user metadata key;
|
||||
/// `AccessDenied` when the decision for `action` depends on a request-body input
|
||||
/// the envelope does not carry (see [`require_body_policy_inputs`]) or when the
|
||||
/// policies deny the request.
|
||||
pub(crate) async fn authorize_owned_request(
|
||||
req: &mut RequestEnvelope,
|
||||
action: Action,
|
||||
location_constraint: Option<String>,
|
||||
) -> rustfs_s3_types::S3Result<()> {
|
||||
prepare_owned_authorization(req, action)?;
|
||||
authorize_request_core(req, action, location_constraint, false)
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Authorizes one exact object key after its enclosing table-catalog operation has been authorized.
|
||||
///
|
||||
/// It applies the same preparation and has the same limits as [`authorize_owned_request`].
|
||||
pub(crate) async fn authorize_owned_internal_object_request(
|
||||
req: &mut RequestEnvelope,
|
||||
action: Action,
|
||||
) -> rustfs_s3_types::S3Result<()> {
|
||||
prepare_owned_authorization(req, action)?;
|
||||
authorize_request_core(req, action, None, true).await.map_err(Into::into)
|
||||
}
|
||||
|
||||
fn prepare_owned_authorization(req: &mut RequestEnvelope, action: Action) -> rustfs_s3_types::S3Result<()> {
|
||||
crate::auth::hoist_signed_query_params(&mut req.headers, req.uri.query()).map_err(rustfs_s3_types::S3Error::from)?;
|
||||
require_body_policy_inputs(req, action)
|
||||
}
|
||||
|
||||
/// Fails closed when the decision for `action` depends on a value the request
|
||||
/// body supplies but the envelope does not carry it. Policy conditions would
|
||||
/// otherwise fall back to client headers, which the stored value need not match
|
||||
/// (header `OFF` with body `ON` would pass a legal-hold Deny).
|
||||
fn require_body_policy_inputs(req: &RequestEnvelope, action: Action) -> rustfs_s3_types::S3Result<()> {
|
||||
let missing = match action {
|
||||
Action::S3Action(S3Action::PutObjectLegalHoldAction) => req.requested_object_legal_hold.is_none(),
|
||||
Action::S3Action(S3Action::PutObjectTaggingAction | S3Action::PutObjectVersionTaggingAction) => {
|
||||
req.requested_object_tags.is_none()
|
||||
}
|
||||
Action::S3Action(S3Action::PutObjectRetentionAction) => req.requested_object_lock_retention.is_none(),
|
||||
_ => false,
|
||||
} || (req.post_object.is_some() && req.post_object_policy_headers.is_none())
|
||||
|| (req.headers.contains_key(COPY_SOURCE_HEADER) && req.requested_copy_source.is_none());
|
||||
if missing {
|
||||
return Err(rustfs_s3_types::s3_error!(
|
||||
AccessDenied,
|
||||
"request body policy input is missing for this action"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const COPY_SOURCE_HEADER: &str = "x-amz-copy-source";
|
||||
|
||||
impl RequestEnvelope {
|
||||
/// The request fields an audit entry for this request is built from.
|
||||
pub(crate) fn audit_view(&self) -> AuditRequestView<'_> {
|
||||
AuditRequestView::from_facts(
|
||||
self.headers(),
|
||||
self.uri(),
|
||||
self.req_info(),
|
||||
self.client_info(),
|
||||
self.remote_addr(),
|
||||
self.request_context(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn authorization_payloads_move_from_extensions_into_named_fields() {
|
||||
let mut extensions = Extensions::new();
|
||||
extensions.insert(MultipartStagingOperation);
|
||||
extensions.insert(StagedMultipartPartAuthorization);
|
||||
let mut retention = RequestedObjectLockRetention(Default::default());
|
||||
retention.0.mode = Some("COMPLIANCE".to_owned().into());
|
||||
extensions.insert(retention);
|
||||
let retain_until = time::OffsetDateTime::from_unix_timestamp(2_000_000_000).expect("valid timestamp");
|
||||
extensions.insert(RequestedObjectLockRetainUntil(retain_until));
|
||||
let head = || RequestHead {
|
||||
method: Method::PUT,
|
||||
uri: "/bucket/key".parse().expect("valid URI"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
};
|
||||
|
||||
let envelope = RequestEnvelope::from_parts(head(), extensions);
|
||||
let empty = RequestEnvelope::from_parts(head(), Extensions::new());
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
envelope.multipart_staging_operation().is_some(),
|
||||
envelope.staged_multipart_part_authorization().is_some(),
|
||||
envelope
|
||||
.requested_object_lock_retention()
|
||||
.and_then(|value| value.0.mode.as_ref().map(|mode| mode.as_str())),
|
||||
envelope.requested_object_lock_retain_until().map(|value| value.0),
|
||||
envelope.residual_extensions().is_empty(),
|
||||
empty.multipart_staging_operation().is_none(),
|
||||
empty.staged_multipart_part_authorization().is_none(),
|
||||
empty.requested_object_lock_retention().is_none(),
|
||||
empty.requested_object_lock_retain_until().is_none(),
|
||||
),
|
||||
(true, true, Some("COMPLIANCE"), Some(retain_until), true, true, true, true, true),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_body_policy_payloads_move_from_extensions_into_named_fields() {
|
||||
let mut form_headers = HeaderMap::new();
|
||||
form_headers.insert("x-amz-tagging", http::HeaderValue::from_static("from=form"));
|
||||
let mut legal_hold = RequestedObjectLegalHold(Default::default());
|
||||
legal_hold.0.status = Some("ON".to_owned().into());
|
||||
let mut tags = RequestedObjectTags(Default::default());
|
||||
tags.0.tag_set.push(Default::default());
|
||||
tags.0.tag_set[0].key = Some("body".to_owned());
|
||||
let mut extensions = Extensions::new();
|
||||
extensions.insert(PostObjectPolicyHeaders(form_headers));
|
||||
extensions.insert(legal_hold);
|
||||
extensions.insert(tags);
|
||||
extensions.insert(RequestedCopySource("source-bucket/source-key".to_owned()));
|
||||
let head = || RequestHead {
|
||||
method: Method::PUT,
|
||||
uri: "/bucket/key".parse().expect("valid URI"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
};
|
||||
|
||||
let envelope = RequestEnvelope::from_parts(head(), extensions);
|
||||
let empty = RequestEnvelope::from_parts(head(), Extensions::new());
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
envelope
|
||||
.post_object_policy_headers()
|
||||
.and_then(|headers| headers.0.get("x-amz-tagging"))
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
envelope
|
||||
.requested_object_legal_hold()
|
||||
.and_then(|hold| hold.0.status.as_ref().map(|status| status.as_str())),
|
||||
envelope
|
||||
.requested_object_tags()
|
||||
.and_then(|tags| tags.0.tag_set.first())
|
||||
.and_then(|tag| tag.key.as_deref()),
|
||||
envelope.requested_copy_source().map(|source| source.0.as_str()),
|
||||
envelope.residual_extensions().is_empty(),
|
||||
empty.post_object_policy_headers().is_none(),
|
||||
empty.requested_object_legal_hold().is_none(),
|
||||
empty.requested_object_tags().is_none(),
|
||||
empty.requested_copy_source().is_none(),
|
||||
),
|
||||
(
|
||||
Some("from=form"),
|
||||
Some("ON"),
|
||||
Some("body"),
|
||||
Some("source-bucket/source-key"),
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_handed_over_secret_is_refused() {
|
||||
assert!(RequestCredentials::new("AKIAENVELOPE".to_owned(), String::new(), None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_absent_secret_matches_no_stored_secret() {
|
||||
let kept = RequestCredentials::without_secret_key("AKIAENVELOPE".to_owned(), None);
|
||||
assert_eq!(kept.secret_key(), None);
|
||||
assert!(!kept.secret_key_matches(""), "absent must not equal an empty stored secret");
|
||||
assert!(!kept.secret_key_matches("stored-secret"));
|
||||
let handed = RequestCredentials::new("AKIAENVELOPE".to_owned(), "stored-secret".to_owned(), None).expect("secret");
|
||||
assert!(!handed.secret_key_matches(""));
|
||||
assert!(!handed.secret_key_matches("other-secret"));
|
||||
assert!(handed.secret_key_matches("stored-secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_reader_of_an_absent_secret_fails_closed() {
|
||||
let kept = RequestCredentials::without_secret_key("AKIAENVELOPE".to_owned(), Some("token".to_owned()));
|
||||
assert!(kept.require_secret_key().is_err());
|
||||
let handed = RequestCredentials::new("AKIAENVELOPE".to_owned(), "s".to_owned(), None).expect("secret");
|
||||
assert_eq!(handed.require_secret_key().ok(), Some("s"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kept_secret_debug_prints_neither_secret_nor_token() {
|
||||
let kept = RequestCredentials::without_secret_key("AKIAENVELOPE".to_owned(), Some("envelope-session-token".to_owned()));
|
||||
let debug = format!("{kept:?}");
|
||||
assert!(debug.contains("AKIAENVELOPE"), "{debug}");
|
||||
assert!(!debug.contains("envelope-session-token"), "{debug}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_credentials_debug_redacts_secret_and_session_token() {
|
||||
let credentials = RequestCredentials::new(
|
||||
"AKIAENVELOPE".to_owned(),
|
||||
"envelope-secret-value".to_owned(),
|
||||
Some("envelope-session-token".to_owned()),
|
||||
)
|
||||
.expect("non-empty secret");
|
||||
);
|
||||
|
||||
let debug = format!("{credentials:?}");
|
||||
|
||||
|
||||
@@ -105,36 +105,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether [`MetadataRoute`] serves this request as a static-website request: the rule
|
||||
/// `is_match` and `call` apply, exposed with s3s-free arguments so the stack dispatcher
|
||||
/// (`server/stack/website.rs`) decides with this same rule rather than a copy of it.
|
||||
pub(crate) fn is_website_request(uri: &Uri, headers: &HeaderMap, domains: &[String]) -> bool {
|
||||
website_target(uri, headers, domains).is_some()
|
||||
}
|
||||
|
||||
/// The bucket-host resolution [`MetadataRoute`] matches the metadata listing extension with,
|
||||
/// held apart from the route so the stack dispatcher (`server/stack/metadata.rs`) can ask the
|
||||
/// same question without naming an s3s type.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct MetadataListingHosts(Option<Arc<MultiDomain>>);
|
||||
|
||||
impl MetadataListingHosts {
|
||||
/// Built from the same virtual-host domain list the route's own [`MultiDomain`] is built from;
|
||||
/// `None` when no server domain is configured, which leaves path-style requests only.
|
||||
pub(crate) fn new(server_domains: Option<&[String]>) -> std::io::Result<Self> {
|
||||
server_domains
|
||||
.map(MultiDomain::new)
|
||||
.transpose()
|
||||
.map(|host| Self(host.map(Arc::new)))
|
||||
.map_err(std::io::Error::other)
|
||||
}
|
||||
|
||||
/// Whether [`MetadataRoute`] serves this request as a `?metadata=true` listing.
|
||||
pub(crate) fn matches(&self, method: &Method, uri: &Uri, headers: &HeaderMap) -> bool {
|
||||
metadata_operation(method, uri, headers, self.0.as_deref()).is_some()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct WebsiteTarget {
|
||||
bucket: String,
|
||||
|
||||
@@ -36,7 +36,6 @@ pub(crate) mod trailer_adapter;
|
||||
// from here rather than through `app::context`.
|
||||
pub(crate) use context::RequestHead;
|
||||
pub use context::{RequestCredentials, RequestEnvelope};
|
||||
pub(crate) use context::{authorize_owned_internal_object_request, authorize_owned_request};
|
||||
|
||||
#[cfg(test)]
|
||||
mod capacity_dirty_scope_test;
|
||||
@@ -49,6 +48,4 @@ pub(crate) mod gating_test_env;
|
||||
#[cfg(test)]
|
||||
mod lifecycle_transition_api_test;
|
||||
#[cfg(test)]
|
||||
mod owned_authorization_test;
|
||||
#[cfg(test)]
|
||||
mod put_prelookup_gating_test;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -53,17 +53,10 @@ pub(crate) mod s3 {
|
||||
PersistedReplicationFilter, PersistedReplicationRule, PersistedReplicationTag,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::access::S3Access;
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::auth::{Credentials, SimpleAuth};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::config::{S3Config, StaticConfigProvider};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::dto::{
|
||||
BucketLocationConstraint, CreateBucketConfiguration, CreateBucketInput, CreateMultipartUploadInput, DeleteObjectInput,
|
||||
ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, PutObjectInput, Tag, Tagging,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::dto::{GetObjectInput, HeadObjectInput, ListObjectsV2Input, ListObjectsV2Output};
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::dto::{ListObjectsInput, StreamingBlob, UploadPartInput, UploadPartOutput};
|
||||
@@ -317,11 +310,10 @@ pub(crate) mod access {
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::ReqInfo;
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{
|
||||
ListBucketsAuthzRequest, PostObjectRequestMarker, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess,
|
||||
TableDataPlaneListCursorPosition, TableDataPlanePublicationGuards, apply_bucket_generation_guard,
|
||||
apply_copy_source_bucket_generation_guard, authorize_copy_object_tags, authorize_request,
|
||||
bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header,
|
||||
load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
|
||||
PostObjectRequestMarker, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, TableDataPlaneListCursorPosition,
|
||||
TableDataPlanePublicationGuards, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard,
|
||||
authorize_copy_object_tags, authorize_request, bucket_config_mutation_incarnation, delete_object_authorize_action,
|
||||
has_bypass_governance_header, load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
|
||||
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller,
|
||||
replication_proxy_allowed, replication_request_authorized, req_info_mut, req_info_ref,
|
||||
retained_table_data_plane_publication_guards, set_requested_object_lock_retain_until,
|
||||
@@ -1673,9 +1665,9 @@ pub(crate) mod timeout_wrapper {
|
||||
pub(crate) mod s3_api {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use crate::storage::storage_api::s3_api_consumer::bucket::{
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_object_versions_output, build_list_objects_output,
|
||||
build_list_objects_v2_output, encode_list_objects_v2_continuation_token, parse_list_object_versions_params,
|
||||
parse_list_objects_v2_params,
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
|
||||
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
|
||||
parse_list_object_versions_params, parse_list_objects_v2_params,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::s3_api_consumer::common::rustfs_owner;
|
||||
}
|
||||
@@ -1711,7 +1703,7 @@ pub(crate) mod bucket_usecase {
|
||||
pub(crate) mod contract {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use super::super::super::storage_contracts::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
|
||||
BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1823,30 +1815,14 @@ pub(crate) mod select_object {
|
||||
};
|
||||
}
|
||||
|
||||
/// The storage items the owned/legacy authorization parity tests drive.
|
||||
#[cfg(test)]
|
||||
pub(crate) mod owned_parity {
|
||||
pub(crate) use crate::storage::storage_api::owned_parity_consumer::{
|
||||
DenialAuditor, DenialRateLimiter, DenialScope, FS, ObjectTagConditions, PostObjectPolicyHeaders, PostObjectRequestMarker,
|
||||
ReqInfo, RequestContext, RequestedCopySource, RequestedObjectLegalHold, RequestedObjectLockRetainUntil,
|
||||
RequestedObjectLockRetention, RequestedObjectTags, SUPPRESSED_DENIALS_TAG, TableDataPlanePublicationGuards,
|
||||
authorization_denial_entry, authorization_denial_entry_for_view, authorize_request, parity_test_support,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod context {
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::EndpointServerPools;
|
||||
pub(crate) use super::bucket;
|
||||
pub(crate) use super::runtime;
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{
|
||||
AuditRequestView, AuthorizationRequest, DenialAuditor, DenialScope, authorize_request_core,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{
|
||||
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, MultipartStagingOperation,
|
||||
ObjectTagConditions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectPolicyHeaders,
|
||||
PostObjectRequestMarker, ReqInfo, RequestedCopySource, RequestedObjectLegalHold, RequestedObjectLockRetainUntil,
|
||||
RequestedObjectLockRetention, RequestedObjectTags, StagedMultipartPartAuthorization, TableDataPlaneListAccess,
|
||||
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, ObjectTagConditions,
|
||||
OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectRequestMarker, ReqInfo, TableDataPlaneListAccess,
|
||||
TableDataPlanePublicationGuards,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::request_context_consumer::RequestContext;
|
||||
|
||||
@@ -78,7 +78,7 @@ impl FromStr for S3Stack {
|
||||
impl S3Stack {
|
||||
/// Unset selects the default; a set but invalid value, including an empty
|
||||
/// one, is a startup error.
|
||||
pub(crate) fn from_env() -> std::io::Result<Self> {
|
||||
fn from_env() -> std::io::Result<Self> {
|
||||
match std::env::var_os(ENV_RUSTFS_S3_STACK) {
|
||||
None => Ok(Self::default()),
|
||||
Some(value) => value
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The access phase of a gateway handler: the legacy access hooks in their owned form.
|
||||
//!
|
||||
//! Responsible for: [`check`], what the legacy access phase runs once per request before any
|
||||
//! operation hook. Per-operation hooks join it here as their operations are registered.
|
||||
//! NOT responsible for: the decisions themselves. Every function here calls the function the
|
||||
//! legacy access phase calls (`storage/access.rs`), so the two stacks cannot answer differently.
|
||||
//! Upstream: each `impl Handler`. Downstream: the storage layer's access functions.
|
||||
//!
|
||||
//! The caller is resolved here on every request, and again by the gateway's `Authorizer` before
|
||||
//! it: the pinned gateway hands a handler no policy snapshot, so one resolution cannot be carried
|
||||
//! from the authorization stages to the handler.
|
||||
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::storage_api::server::http::access::{admit_operation, resolve_req_info};
|
||||
use rustfs_s3_types::S3Result;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Runs the per-request access check for `operation` and installs the caller in `envelope`.
|
||||
///
|
||||
/// The caller is the credential IAM holds for the access key the gateway verified, checked
|
||||
/// against the presented session token; an anonymous request resolves to no caller. The signed
|
||||
/// upload-size capabilities are then refused on any operation but their own, and the license is
|
||||
/// checked.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// What the legacy access phase answers: the credential refusals of the key lookup,
|
||||
/// `InvalidRequest` for a misplaced capability query, `AccessDenied` for a refused license, and
|
||||
/// `InternalError` when IAM or the license cannot be read.
|
||||
pub(crate) async fn check(
|
||||
server_ctx: &Arc<ServerContextSlot>,
|
||||
envelope: &mut RequestEnvelope,
|
||||
operation: &'static str,
|
||||
) -> S3Result<()> {
|
||||
let credentials = envelope.credentials();
|
||||
let access_key = credentials.map(|credentials| credentials.access_key().to_owned());
|
||||
let session_token = credentials
|
||||
.and_then(|credentials| credentials.session_token())
|
||||
.unwrap_or_default()
|
||||
.to_owned();
|
||||
let request_context = envelope.request_context().cloned();
|
||||
let req_info = resolve_req_info(server_ctx, access_key.as_deref(), &session_token, request_context).await?;
|
||||
envelope.install_req_info(req_info);
|
||||
admit_operation(envelope.headers(), envelope.uri().query(), operation)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! `ListBuckets` on the gateway stack.
|
||||
//!
|
||||
//! Responsible for: converting one accepted request into the envelope and input the RustFS use
|
||||
//! case takes, and its answer back.
|
||||
//! NOT responsible for: deciding which buckets the caller sees. The legacy access hook for this
|
||||
//! operation is empty and so is this one: `DefaultBucketUsecase::execute_list_buckets` decides
|
||||
//! per caller (every bucket with `s3:ListAllMyBuckets`, otherwise the buckets the caller may
|
||||
//! list or locate, `AccessDenied` when there is none), for both stacks.
|
||||
//! Upstream: the gateway's dispatch. Downstream: `crate::app::bucket_usecase`.
|
||||
|
||||
use super::super::GatewayBackend;
|
||||
use super::super::access;
|
||||
use super::super::error::handler_error;
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use rustfs_gateway::{Handler, HandlerResult, Operation, Req, Resp, dto};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
|
||||
impl Handler<dto::ListBuckets> for GatewayBackend {
|
||||
async fn call(&self, request: Req<dto::ListBuckets>) -> HandlerResult<dto::ListBuckets> {
|
||||
record_s3_op(S3Operation::ListBuckets);
|
||||
let mut envelope = RequestEnvelope::from_gateway(request.context(), self.server_ctx()).map_err(handler_error)?;
|
||||
access::check(self.server_ctx(), &mut envelope, <dto::ListBuckets as Operation>::NAME)
|
||||
.await
|
||||
.map_err(handler_error)?;
|
||||
let usecase = DefaultBucketUsecase::with_context(self.server_ctx().app_context());
|
||||
let output = usecase
|
||||
.execute_list_buckets(envelope, request.into_input())
|
||||
.await
|
||||
.map_err(handler_error)?;
|
||||
Ok(Resp::new(output))
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Bucket-level operations of the gateway stack, one file each.
|
||||
|
||||
mod list_buckets;
|
||||
@@ -1,357 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! Builds a [`RequestEnvelope`] from a request the gateway accepted, authenticated and routed.
|
||||
//!
|
||||
//! Responsible for: `RequestEnvelope::from_gateway`, the gateway edge's counterpart of the
|
||||
//! legacy edge's `from_s3s` (`storage/s3_compat/envelope.rs`).
|
||||
//! NOT responsible for: the facts the access phase attaches afterwards (`ReqInfo` and what the
|
||||
//! operation hooks retain); `super::access` installs those.
|
||||
//! Upstream: the gateway's handler request context. Downstream: every gateway handler.
|
||||
//!
|
||||
//! Every fact is copied from where the gateway or a RustFS layer in front of it put it: the
|
||||
//! request line and the accepted header lines from the context, the caller from the verified
|
||||
//! principal, and the payloads the connection and request layers attached from the transport
|
||||
//! values the gateway carried through unchanged.
|
||||
|
||||
use crate::app::object::request_body::BodyReadControl;
|
||||
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead};
|
||||
use crate::auth::{
|
||||
AuthType, VerifiedPresignedRequest, VerifiedSigV4Request, get_request_auth_type_with_query, get_session_token,
|
||||
};
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::shared_types::{NativeTlsTransport, RemoteAddr};
|
||||
use crate::storage_api::server::http::request_context::RequestContext;
|
||||
use http::{Extensions, HeaderMap, Uri};
|
||||
use rustfs_gateway::{RequestContextView, TransportExtensions};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::sync::Arc;
|
||||
|
||||
impl RequestEnvelope {
|
||||
/// Copies every request fact of `context` the app layer reads, for the server `server_ctx`.
|
||||
///
|
||||
/// A payload the request does not carry leaves its field `None`. The caller's secret key is
|
||||
/// copied only when the gateway handed it to this operation; the signature markers are set
|
||||
/// only for a caller the gateway verified.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// `InternalError` when the accepted request target cannot be read back as a URI, or a
|
||||
/// handed-over secret key is not text. Neither is the caller's fault.
|
||||
pub(crate) fn from_gateway(context: &RequestContextView, server_ctx: &Arc<ServerContextSlot>) -> S3Result<Self> {
|
||||
let uri = request_uri(context)?;
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in context.headers().iter_raw() {
|
||||
headers.append(name.clone(), value.clone());
|
||||
}
|
||||
let credentials = match context.principal() {
|
||||
Some(principal) => {
|
||||
let secret_key = principal
|
||||
.secret_key_from_authenticator_lookup()
|
||||
.map(|secret| secret.expose_secret());
|
||||
Some(request_credentials(
|
||||
principal.access_key_id(),
|
||||
secret_key,
|
||||
get_session_token(&uri, &headers),
|
||||
)?)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let mut extensions = Extensions::new();
|
||||
let transport = context.transport_extensions();
|
||||
copy_payload::<Option<RemoteAddr>>(transport, &mut extensions);
|
||||
copy_payload::<ClientInfo>(transport, &mut extensions);
|
||||
copy_payload::<NativeTlsTransport>(transport, &mut extensions);
|
||||
copy_payload::<RequestContext>(transport, &mut extensions);
|
||||
copy_payload::<BodyReadControl>(transport, &mut extensions);
|
||||
extensions.insert(Arc::clone(server_ctx));
|
||||
// The legacy access phase marks a request whose SigV4 signature its edge verified; here
|
||||
// the gateway verified it, and only a verified caller has a principal.
|
||||
if credentials.is_some() {
|
||||
match get_request_auth_type_with_query(&headers, uri.query()) {
|
||||
AuthType::Presigned => {
|
||||
extensions.insert(VerifiedPresignedRequest);
|
||||
extensions.insert(VerifiedSigV4Request);
|
||||
}
|
||||
AuthType::Signed => {
|
||||
extensions.insert(VerifiedSigV4Request);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let scope = context.verified_scope();
|
||||
let head = RequestHead {
|
||||
method: context.method().clone(),
|
||||
uri,
|
||||
headers,
|
||||
credentials,
|
||||
region: scope.map(|scope| scope.region().to_owned()),
|
||||
service: scope.map(|scope| scope.service().to_owned()),
|
||||
};
|
||||
Ok(Self::from_parts(head, extensions))
|
||||
}
|
||||
}
|
||||
|
||||
/// The request target as it arrived: the raw path and query, with the scheme and authority when
|
||||
/// the transport carried them (an absolute-form target, or HTTP/2).
|
||||
fn request_uri(context: &RequestContextView) -> S3Result<Uri> {
|
||||
let path_and_query = match context.raw_query() {
|
||||
"" => context.raw_path().to_owned(),
|
||||
query => format!("{}?{query}", context.raw_path()),
|
||||
};
|
||||
let builder = match (context.target_scheme(), context.target_authority()) {
|
||||
(Some(scheme), Some(authority)) => Uri::builder().scheme(scheme).authority(authority),
|
||||
_ => Uri::builder(),
|
||||
};
|
||||
builder
|
||||
.path_and_query(path_and_query)
|
||||
.build()
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "the accepted request target is not a URI"))
|
||||
}
|
||||
|
||||
/// The caller's credentials as the app layer holds them. `secret_key` is what the gateway handed
|
||||
/// over, which is nothing unless the operation opted in.
|
||||
fn request_credentials(access_key: &str, secret_key: Option<&[u8]>, session_token: Option<&str>) -> S3Result<RequestCredentials> {
|
||||
let session_token = session_token.map(str::to_owned);
|
||||
let Some(secret_key) = secret_key else {
|
||||
return Ok(RequestCredentials::without_secret_key(access_key.to_owned(), session_token));
|
||||
};
|
||||
let secret_key = std::str::from_utf8(secret_key)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "the caller's secret key is not text"))?;
|
||||
// The gateway refuses a credential whose secret is empty, so a handed-over secret is never
|
||||
// empty. Should one arrive, the request is refused rather than served with a secret that
|
||||
// every stored secret would have to be compared against.
|
||||
RequestCredentials::new(access_key.to_owned(), secret_key.to_owned(), session_token)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InternalError, "the caller's secret key is empty"))
|
||||
}
|
||||
|
||||
/// Copies the payload of type `T` a layer in front of the gateway attached, when one did.
|
||||
fn copy_payload<T: Clone + Send + Sync + 'static>(transport: &TransportExtensions, extensions: &mut Extensions) {
|
||||
if let Some(payload) = transport.get::<T>() {
|
||||
extensions.insert(payload.clone());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::server::gateway_handlers::error::handler_error;
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderValue, Method, Request, StatusCode};
|
||||
use http_body_util::Empty;
|
||||
use rustfs_gateway::sig::{AmzDate, PayloadMode, SigService, SigV4Signer, SigningCredentials, SigningRequest, SigningScope};
|
||||
use rustfs_gateway::{
|
||||
Credentials, Handler, HandlerResult, Limits, RegionSet, Req, Resp, ServiceBuilder, SigV4Authenticator, StaticCredentials,
|
||||
WireRequest, allow_when, dto,
|
||||
};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Mutex;
|
||||
use std::time::Instant;
|
||||
|
||||
const PEER: ([u8; 4], u16) = ([10, 0, 0, 7], 41000);
|
||||
const CLIENT: [u8; 4] = [203, 0, 113, 9];
|
||||
const REQUEST_ID: &str = "envelope-request-id";
|
||||
const SIGNER_ACCESS_KEY: &str = "AKIAENVELOPESIGNER";
|
||||
const SIGNER_SECRET_KEY: &str = "envelope-signer-secret-key-0123456789";
|
||||
|
||||
/// Records the envelope the gateway edge builds for the request it is handed.
|
||||
struct Probe {
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
seen: Mutex<Option<RequestEnvelope>>,
|
||||
}
|
||||
|
||||
impl Handler<dto::ListBuckets> for Probe {
|
||||
async fn call(&self, request: Req<dto::ListBuckets>) -> HandlerResult<dto::ListBuckets> {
|
||||
let envelope = RequestEnvelope::from_gateway(request.context(), &self.server_ctx).map_err(handler_error)?;
|
||||
*self.seen.lock().expect("probe lock") = Some(envelope);
|
||||
Ok(Resp::new(dto::ListBucketsOutput::default()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends `request` through a gateway whose only handler is the probe, and returns what the
|
||||
/// probe built. The test authorizer admits the one probed operation and nothing else.
|
||||
async fn envelope_of(request: Request<Empty<Bytes>>, server_ctx: &Arc<ServerContextSlot>) -> RequestEnvelope {
|
||||
let probe = Arc::new(Probe {
|
||||
server_ctx: Arc::clone(server_ctx),
|
||||
seen: Mutex::new(None),
|
||||
});
|
||||
let regions = RegionSet::new(["us-east-1"]).expect("a region");
|
||||
let service = ServiceBuilder::new()
|
||||
.rustfs_profile()
|
||||
.authenticator(SigV4Authenticator::new(Arc::new(signer_credentials()), regions).rustfs_profile())
|
||||
.authorizer(allow_when(|request| request.operation == "ListBuckets"))
|
||||
.register::<dto::ListBuckets, _>(Arc::clone(&probe))
|
||||
.build()
|
||||
.expect("the probe service assembles");
|
||||
|
||||
let response = service.call(request).await;
|
||||
assert_eq!(response.status(), StatusCode::OK, "the probe must be reached");
|
||||
probe
|
||||
.seen
|
||||
.lock()
|
||||
.expect("probe lock")
|
||||
.take()
|
||||
.expect("the probe saw the request")
|
||||
}
|
||||
|
||||
/// The one caller the probe service knows.
|
||||
fn signer_credentials() -> StaticCredentials {
|
||||
StaticCredentials::new().with(Credentials::new(SIGNER_ACCESS_KEY, SIGNER_SECRET_KEY.as_bytes()).expect("a valid key"))
|
||||
}
|
||||
|
||||
/// A ListBuckets request header-signed now by the probe service's one caller.
|
||||
fn signed_list_buckets() -> Request<Empty<Bytes>> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(http::header::HOST, HeaderValue::from_static("localhost"));
|
||||
let probe = list_buckets_request("/").body(Bytes::new()).expect("static request");
|
||||
let accepted = WireRequest::accept(probe, &Limits::default()).expect("an acceptable host");
|
||||
let stamp = AmzDate::parse(&chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string()).expect("a stamp");
|
||||
let scope = SigningScope::new(stamp.day(), "us-east-1", SigService::S3).expect("a scope");
|
||||
let credentials = SigningCredentials::new(SIGNER_ACCESS_KEY, SIGNER_SECRET_KEY.as_bytes()).expect("credentials");
|
||||
let mut signer = SigV4Signer::new(credentials, scope);
|
||||
let method = Method::GET;
|
||||
let signing =
|
||||
SigningRequest::new(&method, "/", "", &headers, accepted.host().raw_for_signing(), PayloadMode::Empty, stamp);
|
||||
let signed = signer.sign_headers(&signing).expect("a signable request");
|
||||
let mut builder = Request::builder().method(method).uri("/");
|
||||
for (name, value) in signed.headers() {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
builder.body(Empty::new()).expect("a request")
|
||||
}
|
||||
|
||||
fn list_buckets_request(target: &str) -> http::request::Builder {
|
||||
Request::builder().method(Method::GET).uri(target).header("host", "localhost")
|
||||
}
|
||||
|
||||
fn request_context() -> RequestContext {
|
||||
RequestContext {
|
||||
request_id: REQUEST_ID.to_owned(),
|
||||
x_amz_request_id: REQUEST_ID.to_owned(),
|
||||
trace_id: None,
|
||||
span_id: None,
|
||||
start_time: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn from_gateway_copies_every_fact_of_the_accepted_request() {
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
let request = list_buckets_request("/?prefix=probe")
|
||||
.header("x-envelope-probe", "probe")
|
||||
.extension(Some(RemoteAddr(SocketAddr::from(PEER))))
|
||||
.extension(ClientInfo::direct(SocketAddr::new(IpAddr::from(CLIENT), 443)))
|
||||
.extension(NativeTlsTransport(true))
|
||||
.extension(request_context())
|
||||
.extension(BodyReadControl::default())
|
||||
.body(Empty::new())
|
||||
.expect("static request");
|
||||
|
||||
let envelope = envelope_of(request, &server_ctx).await;
|
||||
|
||||
assert_eq!(envelope.method(), Method::GET);
|
||||
assert_eq!(envelope.uri().path(), "/");
|
||||
assert_eq!(envelope.uri().query(), Some("prefix=probe"));
|
||||
assert_eq!(envelope.headers().get("x-envelope-probe"), Some(&HeaderValue::from_static("probe")));
|
||||
assert_eq!(envelope.remote_addr(), Some(SocketAddr::from(PEER)));
|
||||
assert_eq!(envelope.client_info().map(|info| info.real_ip), Some(IpAddr::from(CLIENT)));
|
||||
assert_eq!(envelope.native_tls_transport(), Some(true));
|
||||
assert_eq!(envelope.request_context().map(|context| context.request_id.as_str()), Some(REQUEST_ID));
|
||||
assert!(envelope.body_read_control().is_some(), "body_read_control");
|
||||
assert!(
|
||||
envelope.server_context().is_some_and(|slot| Arc::ptr_eq(slot, &server_ctx)),
|
||||
"server_context"
|
||||
);
|
||||
assert!(envelope.residual_extensions().is_empty(), "every copied payload is a named field");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn from_gateway_leaves_absent_payloads_none_and_an_anonymous_caller_unverified() {
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
let request = list_buckets_request("/").body(Empty::new()).expect("static request");
|
||||
|
||||
let envelope = envelope_of(request, &server_ctx).await;
|
||||
|
||||
assert!(envelope.credentials().is_none(), "credentials");
|
||||
assert_eq!(envelope.region(), None, "region");
|
||||
assert_eq!(envelope.service(), None, "service");
|
||||
assert_eq!(envelope.remote_addr(), None, "remote_addr");
|
||||
assert!(envelope.client_info().is_none(), "client_info");
|
||||
assert_eq!(envelope.native_tls_transport(), None, "native_tls_transport");
|
||||
assert!(envelope.request_context().is_none(), "request_context");
|
||||
assert!(envelope.body_read_control().is_none(), "body_read_control");
|
||||
assert!(envelope.verified_sigv4().is_none(), "verified_sigv4");
|
||||
assert!(envelope.verified_presigned().is_none(), "verified_presigned");
|
||||
assert!(envelope.req_info().is_none(), "the access phase installs req_info, not the edge");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn from_gateway_treats_a_peerless_remote_addr_as_absent() {
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
let request = list_buckets_request("/")
|
||||
.extension(None::<RemoteAddr>)
|
||||
.body(Empty::new())
|
||||
.expect("static request");
|
||||
|
||||
let envelope = envelope_of(request, &server_ctx).await;
|
||||
|
||||
assert_eq!(envelope.remote_addr(), None);
|
||||
assert!(envelope.residual_extensions().is_empty(), "the peerless marker is consumed, not kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credentials_carry_the_secret_only_when_the_gateway_handed_it_over() {
|
||||
let kept = request_credentials("AKIAENVELOPE", None, Some("token")).expect("credentials");
|
||||
assert_eq!(kept.access_key(), "AKIAENVELOPE");
|
||||
assert!(kept.secret_key().is_none());
|
||||
assert_eq!(kept.session_token(), Some("token"));
|
||||
|
||||
let handed = request_credentials("AKIAENVELOPE", Some(b"envelope-secret"), None).expect("credentials");
|
||||
assert_eq!(handed.secret_key(), Some("envelope-secret"));
|
||||
assert_eq!(handed.session_token(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_handed_over_secret_is_refused_instead_of_kept() {
|
||||
let error = request_credentials("AKIAENVELOPE", Some(b""), None).expect_err("empty");
|
||||
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
/// ListBuckets does not opt in to the caller's secret, so the gateway keeps it: the handler
|
||||
/// sees who signed and no secret, and a reader that needs one is refused instead of being
|
||||
/// handed an empty string to compare with.
|
||||
#[tokio::test]
|
||||
async fn an_operation_that_did_not_opt_in_reaches_the_handler_without_the_caller_s_secret() {
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
|
||||
let envelope = envelope_of(signed_list_buckets(), &server_ctx).await;
|
||||
|
||||
let credentials = envelope.credentials().expect("a signed request has a caller");
|
||||
assert_eq!(credentials.access_key(), SIGNER_ACCESS_KEY);
|
||||
assert!(credentials.secret_key().is_none(), "the secret stays with the gateway");
|
||||
assert!(credentials.require_secret_key().is_err(), "a reader of the secret fails closed");
|
||||
assert!(!credentials.secret_key_matches(""), "an absent secret equals no stored secret");
|
||||
assert!(!credentials.secret_key_matches(SIGNER_SECRET_KEY));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_secret_that_is_not_text_is_refused_instead_of_dropped() {
|
||||
let error = request_credentials("AKIAENVELOPE", Some(&[0xff, 0xfe]), None).expect_err("not text");
|
||||
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! The one conversion of a RustFS error into the gateway's handler error.
|
||||
//!
|
||||
//! Responsible for: [`handler_error`], called by every gateway handler and by nothing else.
|
||||
//! NOT responsible for: choosing an error (the use case's), or writing it (the gateway renders the
|
||||
//! document, the status and the request identifiers).
|
||||
//! Upstream: `rustfs_s3_types::S3Error`. Downstream: `rustfs_gateway::HandlerError`.
|
||||
//!
|
||||
//! The gateway's error carries a code with its status, a message, and response headers from a
|
||||
//! closed set. An error this function cannot carry whole is answered `500 InternalError` rather
|
||||
//! than sent with a member missing: a `416` without its `Content-Range`, or a throttle without
|
||||
//! its `Retry-After`, tells a client something RustFS did not say. Each operation slice that
|
||||
//! raises a header not listed in [`error_header`] adds it there, with its test.
|
||||
|
||||
use http::{HeaderName, HeaderValue, StatusCode};
|
||||
use rustfs_gateway::{ErrorHeader, HandlerError};
|
||||
use rustfs_gateway_types::ErrorCode;
|
||||
use rustfs_s3_types::S3Error;
|
||||
use std::borrow::Cow;
|
||||
|
||||
/// Answered when an error names a code without a status: the raiser of a custom code must set one.
|
||||
const MISSING_STATUS: &str = "the error carries a code without a status";
|
||||
/// Answered when an error carries a response header this edge cannot write.
|
||||
const UNWRITABLE_HEADER: &str = "the error carries a response header the gateway cannot write";
|
||||
|
||||
/// Converts a use-case error into the error the gateway answers with: the same code, status and
|
||||
/// message, and every response header it carries.
|
||||
pub(crate) fn handler_error(error: S3Error) -> HandlerError {
|
||||
let parts = error.into_parts();
|
||||
let Some(status) = parts.status.or_else(|| parts.code.status_code()) else {
|
||||
return HandlerError::internal_error(MISSING_STATUS);
|
||||
};
|
||||
let mut mapped = HandlerError::new(error_code(parts.code.as_str(), status), parts.message.unwrap_or(Cow::Borrowed("")));
|
||||
for (name, value) in parts.headers.iter().flatten() {
|
||||
let Some(header) = error_header(name, value) else {
|
||||
return HandlerError::internal_error(UNWRITABLE_HEADER);
|
||||
};
|
||||
mapped = mapped.with_header(header);
|
||||
}
|
||||
mapped
|
||||
}
|
||||
|
||||
/// The gateway's code for `name` answered with `status`: its own declared code when both agree,
|
||||
/// otherwise a code with exactly the spelling and status RustFS chose.
|
||||
fn error_code(name: &str, status: StatusCode) -> ErrorCode {
|
||||
match ErrorCode::known(name) {
|
||||
Some(declared) if declared.default_status() == status => declared,
|
||||
_ => ErrorCode::custom(name.to_owned(), status),
|
||||
}
|
||||
}
|
||||
|
||||
/// The gateway's spelling of one error response header, or `None` for a header it cannot write.
|
||||
fn error_header(name: &HeaderName, value: &HeaderValue) -> Option<ErrorHeader> {
|
||||
if name == http::header::RETRY_AFTER {
|
||||
let seconds = value.to_str().ok()?.parse().ok()?;
|
||||
return Some(ErrorHeader::RetryAfter { seconds });
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::HeaderMap;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
|
||||
#[test]
|
||||
fn a_declared_code_keeps_its_spelling_status_and_message() {
|
||||
let mapped = handler_error(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "AccessDenied");
|
||||
assert_eq!(mapped.code().default_status(), StatusCode::FORBIDDEN);
|
||||
assert_eq!(mapped.message(), "Access Denied");
|
||||
assert!(mapped.headers().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_internal_error_stays_a_server_fault() {
|
||||
let mapped = handler_error(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "InternalError");
|
||||
assert_eq!(mapped.code().default_status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert_eq!(mapped.message(), "Not init");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_explicit_status_overrides_the_declared_one() {
|
||||
let mut error = S3Error::with_message(S3ErrorCode::AccessDenied, "locked");
|
||||
error.set_status_code(StatusCode::CONFLICT);
|
||||
|
||||
let mapped = handler_error(error);
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "AccessDenied");
|
||||
assert_eq!(mapped.code().default_status(), StatusCode::CONFLICT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_custom_code_keeps_the_status_its_raiser_set() {
|
||||
let mut error = S3Error::with_message(S3ErrorCode::Custom("RustFsOnly".into()), "custom");
|
||||
error.set_status_code(StatusCode::SERVICE_UNAVAILABLE);
|
||||
|
||||
let mapped = handler_error(error);
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "RustFsOnly");
|
||||
assert_eq!(mapped.code().default_status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(mapped.message(), "custom");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_custom_code_without_a_status_is_a_server_fault_not_a_guess() {
|
||||
let mapped = handler_error(S3Error::with_message(S3ErrorCode::Custom("RustFsOnly".into()), "custom"));
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "InternalError");
|
||||
assert_eq!(mapped.code().default_status(), StatusCode::INTERNAL_SERVER_ERROR);
|
||||
assert_eq!(mapped.message(), MISSING_STATUS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_message_is_not_invented() {
|
||||
let mapped = handler_error(S3Error::new(S3ErrorCode::AccessDenied));
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "AccessDenied");
|
||||
assert_eq!(mapped.message(), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_retry_after_header_is_carried() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(http::header::RETRY_AFTER, HeaderValue::from_static("7"));
|
||||
let mut error = S3Error::with_message(S3ErrorCode::SlowDown, "busy");
|
||||
error.set_headers(headers);
|
||||
|
||||
let mapped = handler_error(error);
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "SlowDown");
|
||||
assert_eq!(mapped.headers(), [ErrorHeader::RetryAfter { seconds: 7 }]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_header_the_gateway_cannot_write_fails_the_whole_error() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("x-rustfs-unlisted", HeaderValue::from_static("value"));
|
||||
let mut error = S3Error::with_message(S3ErrorCode::AccessDenied, "denied");
|
||||
error.set_headers(headers);
|
||||
|
||||
let mapped = handler_error(error);
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "InternalError");
|
||||
assert_eq!(mapped.message(), UNWRITABLE_HEADER);
|
||||
assert!(mapped.headers().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreadable_retry_after_fails_the_whole_error() {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(http::header::RETRY_AFTER, HeaderValue::from_static("soon"));
|
||||
let mut error = S3Error::with_message(S3ErrorCode::SlowDown, "busy");
|
||||
error.set_headers(headers);
|
||||
|
||||
assert_eq!(handler_error(error).message(), UNWRITABLE_HEADER);
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The S3 operations the gateway stack serves, one file per operation (rustfs/backlog#2734).
|
||||
//!
|
||||
//! Responsible for: [`GatewayBackend`], the one backend every operation is implemented on, and
|
||||
//! [`register`], the explicit list of what is registered. An operation that is not in that list
|
||||
//! has no handler, and the gateway answers it `501 NotImplemented`.
|
||||
//! NOT responsible for: assembling the service (`server/gateway_stack.rs`), authenticating or
|
||||
//! routing a request (the gateway), or any S3 behaviour: a handler converts and delegates, and
|
||||
//! the RustFS use case it calls is the one the legacy stack calls.
|
||||
//! Upstream: the gateway's dispatch. Downstream: `crate::app` use cases.
|
||||
//!
|
||||
//! # The shape every handler has
|
||||
//!
|
||||
//! `<family>/<snake_op>.rs` holds exactly one `impl Handler<Op> for GatewayBackend`, in four
|
||||
//! steps and nothing else:
|
||||
//!
|
||||
//! 1. build the [`RequestEnvelope`](crate::app::RequestEnvelope) with `from_gateway`;
|
||||
//! 2. run the access phase: [`access::check`], then the operation's own access hook, which is
|
||||
//! the authoritative authorization and runs exactly where the legacy stack runs it;
|
||||
//! 3. call the use case with the envelope and the gateway's own input type;
|
||||
//! 4. map a refusal with [`error::handler_error`], the only place an error is converted.
|
||||
//!
|
||||
//! The gateway's `Authorizer` (`server/gateway_stack/authorizer.rs`) is an earlier gate in front
|
||||
//! of step 2 and never a replacement for it.
|
||||
|
||||
pub(crate) mod access;
|
||||
mod bucket;
|
||||
mod envelope;
|
||||
pub(crate) mod error;
|
||||
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use rustfs_gateway::{ServiceBuilder, dto};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The backend every gateway operation is implemented on: the server whose use cases answer.
|
||||
pub(crate) struct GatewayBackend {
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
}
|
||||
|
||||
impl GatewayBackend {
|
||||
pub(crate) fn new(server_ctx: Arc<ServerContextSlot>) -> Self {
|
||||
Self { server_ctx }
|
||||
}
|
||||
|
||||
pub(crate) fn server_ctx(&self) -> &Arc<ServerContextSlot> {
|
||||
&self.server_ctx
|
||||
}
|
||||
}
|
||||
|
||||
/// Registers every operation the gateway stack serves. Each slice adds one line here and one row
|
||||
/// to the authorization table; the census in `server/gateway_stack/authorizer.rs` holds the two
|
||||
/// lists equal.
|
||||
pub(crate) fn register(builder: ServiceBuilder, backend: &Arc<GatewayBackend>) -> ServiceBuilder {
|
||||
builder.register::<dto::ListBuckets, _>(Arc::clone(backend))
|
||||
}
|
||||
@@ -1,315 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! The gateway S3 stack of one server: its assembly, and the facts the host hands every request
|
||||
//! (rustfs/backlog#2734 T2.5).
|
||||
//!
|
||||
//! Responsible for: [`GatewayStack::assemble`], the one place the `rustfs-gateway` service is
|
||||
//! built for RustFS — the RustFS profile, signature verification against RustFS IAM, the RustFS
|
||||
//! authorizer, explicit deadlines, and the registered handlers — and [`GatewayStack::call`],
|
||||
//! which attaches the host's facts to a request before the gateway accepts it.
|
||||
//! NOT responsible for: choosing the stack (`server/stack.rs`), the middleware around it
|
||||
//! (`server/http.rs`), or any S3 behaviour (`server/gateway_handlers` and the use cases).
|
||||
//! Upstream: `server/http.rs`. Downstream: `rustfs-gateway`, `rustfs-gateway-adapter`.
|
||||
//!
|
||||
//! # What the host keeps
|
||||
//!
|
||||
//! RustFS's own layers stay in front of the gateway and the gateway's counterparts stay off
|
||||
//! (rustfs/backlog#2771): CORS is answered by `ConditionalCorsLayer`, so no CORS source is
|
||||
//! installed and the gateway writes no CORS header; the SSE-C transport layer runs first and the
|
||||
//! gateway's own strict gate stays behind it; and the request identifier is the one
|
||||
//! `ExternalRequestContextLayer` assigned, handed over as the gateway's `HostRequestId`, so an
|
||||
//! answer and RustFS's logs name a request identically.
|
||||
|
||||
// The stack is under construction, so a build without debug assertions must not carry it. The
|
||||
// message is checked by `scripts/check_gateway_stack_release_boundary.sh`.
|
||||
#[cfg(not(debug_assertions))]
|
||||
compile_error!("gateway-stack is restricted to debug builds");
|
||||
|
||||
mod authorizer;
|
||||
|
||||
use self::authorizer::{AUTHORIZATION_TABLE, OwnedDecisionCore, RequestLine, RustfsAuthorizer};
|
||||
use crate::runtime_sources::{ServerContextSlot, current_action_credentials};
|
||||
use crate::server::gateway_handlers::{GatewayBackend, register};
|
||||
use crate::server::ssec_transport::is_secure_transport;
|
||||
use crate::shared_types::{NativeTlsTransport, RemoteAddr};
|
||||
use crate::storage_api::server::http::request_context::RequestContext;
|
||||
use http::{Extensions, Request, Response};
|
||||
use rustfs_config::RUSTFS_REGION;
|
||||
use rustfs_gateway::{
|
||||
AssemblyError, Authorizer, BoxFuture, ClientAddr, DEFAULT_MAX_BUFFERED_BODY_BYTES, HandlerDeadlineClass,
|
||||
HandlerDeadlineConfig, HostRequestId, LegacyRustfsVirtualHosts, MintedTraces, RegionSet, RequestBodyDeadlineConfig,
|
||||
ServiceBuilder, ServiceConfig, SigV4Authenticator, SseConfig, TransportSecurity,
|
||||
};
|
||||
use rustfs_gateway_adapter::{IamAdapters, IamKeyLookup, IamPrincipalLookup, RootCredentials, RootCredentialsError, RootSource};
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::manager::get_token_signing_key;
|
||||
use rustfs_iam::store::object::ObjectStore;
|
||||
use rustfs_iam::sys::IamSys;
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
/// What the assembly takes from the server it is built for.
|
||||
pub(crate) struct GatewayAssembly {
|
||||
pub(crate) server_ctx: Arc<ServerContextSlot>,
|
||||
/// The configured root credentials, checked once here. Requests read the live ones.
|
||||
pub(crate) root_access_key: String,
|
||||
pub(crate) root_secret_key: String,
|
||||
/// The configured region; the default region signs when none is configured.
|
||||
pub(crate) region: Option<String>,
|
||||
/// The virtual-hosted-style base domains of the S3 listener, as the legacy stack reads them.
|
||||
pub(crate) server_domains: Vec<String>,
|
||||
/// `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT`: the longest silence between two body reads. Zero
|
||||
/// disables the bound, as it does on the legacy stack.
|
||||
pub(crate) body_read_timeout: Duration,
|
||||
/// Whether Keystone authentication is enabled in configuration (`RUSTFS_KEYSTONE_ENABLE`).
|
||||
pub(crate) keystone_enabled: bool,
|
||||
}
|
||||
|
||||
/// Whether Keystone authentication is switched on in configuration. A Keystone configuration
|
||||
/// that cannot be read counts as switched on: only a readable configuration can say it is off.
|
||||
pub(crate) fn keystone_enabled_in_configuration() -> bool {
|
||||
rustfs_keystone::KeystoneConfig::from_env().map_or(true, |config| config.enable)
|
||||
}
|
||||
|
||||
/// Why the gateway stack could not be assembled. Startup fails with it; the legacy stack is never
|
||||
/// started in its place.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum GatewayAssemblyError {
|
||||
#[error("the root credentials cannot verify signatures on the gateway stack: {0}")]
|
||||
RootCredentials(RootCredentialsError),
|
||||
#[error("region {region:?} cannot be a signing region on the gateway stack")]
|
||||
Region { region: String },
|
||||
#[error("the server domains cannot be served by the gateway stack: {0:?}")]
|
||||
ServerDomains(rustfs_gateway::LegacyDomainError),
|
||||
#[error("the gateway service refused its assembly: {0:?}")]
|
||||
Service(AssemblyError),
|
||||
#[error(
|
||||
"RUSTFS_S3_STACK=gateway cannot be served while RUSTFS_KEYSTONE_ENABLE is on: the gateway admits a request that \
|
||||
carries only X-Auth-Token as anonymous while RustFS reads the Keystone identity behind it; Keystone support on the \
|
||||
gateway stack is rustfs/gateway#1586"
|
||||
)]
|
||||
KeystoneEnabled,
|
||||
}
|
||||
|
||||
/// The assembled gateway service.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct GatewayStack {
|
||||
service: rustfs_gateway::S3Service,
|
||||
}
|
||||
|
||||
impl GatewayStack {
|
||||
/// Assembles the gateway stack for one server.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`GatewayAssemblyError`] naming the input the gateway refused. The secret key is never
|
||||
/// part of it.
|
||||
pub(crate) fn assemble(assembly: GatewayAssembly) -> Result<Self, GatewayAssemblyError> {
|
||||
let iam = Arc::new(ServerIam {
|
||||
server_ctx: Arc::clone(&assembly.server_ctx),
|
||||
});
|
||||
let root = Arc::new(live_root(Arc::clone(&assembly.server_ctx)));
|
||||
let core = OwnedDecisionCore::new(Arc::clone(&assembly.server_ctx));
|
||||
let authorizer = RustfsAuthorizer::new(AUTHORIZATION_TABLE, Arc::new(core));
|
||||
Self::assemble_with(assembly, iam, root, authorizer)
|
||||
}
|
||||
|
||||
fn assemble_with(
|
||||
assembly: GatewayAssembly,
|
||||
iam: Arc<dyn IamPrincipalLookup>,
|
||||
root: Arc<dyn RootSource>,
|
||||
authorizer: impl Authorizer,
|
||||
) -> Result<Self, GatewayAssemblyError> {
|
||||
// The pinned gateway admits a request that carries only `X-Auth-Token` as anonymous and
|
||||
// runs no authenticator for it, while the RustFS code behind the gateway reads the
|
||||
// Keystone identity: the request would be authorized as one caller and served as
|
||||
// another. Refused until the gateway authenticates the token (rustfs/gateway#1586).
|
||||
if assembly.keystone_enabled {
|
||||
return Err(GatewayAssemblyError::KeystoneEnabled);
|
||||
}
|
||||
// A root the gateway could not verify a signature with is a startup error, never a
|
||||
// server that silently has no root. Requests read the live root from `root`.
|
||||
RootCredentials::new(&assembly.root_access_key, &assembly.root_secret_key, &assembly.root_secret_key)
|
||||
.map_err(GatewayAssemblyError::RootCredentials)?;
|
||||
let region = assembly.region.as_deref().unwrap_or(RUSTFS_REGION);
|
||||
let regions = RegionSet::new([region]).map_err(|_| GatewayAssemblyError::Region {
|
||||
region: region.to_owned(),
|
||||
})?;
|
||||
let hosts = LegacyRustfsVirtualHosts::new(&assembly.server_domains).map_err(GatewayAssemblyError::ServerDomains)?;
|
||||
let backend = Arc::new(GatewayBackend::new(Arc::clone(&assembly.server_ctx)));
|
||||
|
||||
// Every legacy RustFS reading of a request, a credential scope and an answer, in the
|
||||
// gateway's one reviewable preset; its other half is on the authenticator below.
|
||||
let builder = ServiceBuilder::new().rustfs_profile();
|
||||
// The credential provider and the policy source go in together, over one IAM and one
|
||||
// live root, so a key cannot be root to one and an ordinary user to the other.
|
||||
let builder = IamAdapters::new(iam, root)
|
||||
.install(builder, |credentials| SigV4Authenticator::new(credentials, regions).rustfs_profile());
|
||||
let builder = builder
|
||||
.authorizer(authorizer)
|
||||
// The outer SSE-C transport layer answers first; the gateway's own gate stays strict
|
||||
// behind it.
|
||||
.sse_config(SseConfig::strict())
|
||||
.host_resolver(hosts)
|
||||
// The fallback only: `attach_host_facts` hands RustFS's own identifier over.
|
||||
.trace_source(MintedTraces::with_uuid_request_ids());
|
||||
let (builder, _settings) = builder.config(service_config(assembly.body_read_timeout));
|
||||
let service = register(builder, &backend).build().map_err(GatewayAssemblyError::Service)?;
|
||||
Ok(Self { service })
|
||||
}
|
||||
|
||||
/// The operations the assembled service answers, read from its registry. Everything else is
|
||||
/// answered `501 NotImplemented`.
|
||||
pub(crate) fn registered_operations(&self) -> Vec<&'static str> {
|
||||
self.service.operations().collect()
|
||||
}
|
||||
|
||||
/// Answers one request on the gateway stack, with the host's facts attached.
|
||||
pub(crate) fn call<B>(&self, mut request: Request<B>) -> impl Future<Output = Response<rustfs_gateway::Body>> + Send + 'static
|
||||
where
|
||||
B: http_body::Body + Send + 'static,
|
||||
B::Data: Send,
|
||||
B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
{
|
||||
// The authorizer is handed the headers and the query but not the request line, and the
|
||||
// shared decision core is asked about the request as it arrived.
|
||||
let line = RequestLine {
|
||||
method: request.method().clone(),
|
||||
uri: request.uri().clone(),
|
||||
};
|
||||
request.extensions_mut().insert(line);
|
||||
attach_host_facts(request.extensions_mut());
|
||||
let mut service = self.service.clone();
|
||||
let answer = tower::Service::call(&mut service, request);
|
||||
async move {
|
||||
match answer.await {
|
||||
Ok(response) => response,
|
||||
Err(never) => match never {},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The gateway's request settings for RustFS, every deadline stated instead of inherited.
|
||||
///
|
||||
/// No handler runs under a total deadline, object data included: the legacy stack has none, and a
|
||||
/// large transfer must not be cut off for its duration. A committed answer is not bounded either.
|
||||
/// The request body keeps RustFS's own idle bound and no throughput floor.
|
||||
fn service_config(body_read_timeout: Duration) -> ServiceConfig {
|
||||
let handlers = HandlerDeadlineConfig::default()
|
||||
.without_deadline(HandlerDeadlineClass::Standard)
|
||||
.without_deadline(HandlerDeadlineClass::Extended)
|
||||
.without_commit_progress_deadline();
|
||||
ServiceConfig::new(DEFAULT_MAX_BUFFERED_BODY_BYTES)
|
||||
.with_handler_deadlines(handlers)
|
||||
.with_request_body_deadlines(body_deadlines(body_read_timeout))
|
||||
}
|
||||
|
||||
/// RustFS's body read timeout as the gateway's idle deadlines: the same silence is allowed before
|
||||
/// the first byte and between reads, and zero lifts both.
|
||||
fn body_deadlines(read_timeout: Duration) -> RequestBodyDeadlineConfig {
|
||||
RequestBodyDeadlineConfig::new(read_timeout, read_timeout)
|
||||
.unwrap_or(RequestBodyDeadlineConfig::S3.without_idle_deadlines())
|
||||
.without_throughput_floor()
|
||||
}
|
||||
|
||||
/// Attaches what the gateway reads about a request from its host: whether the transport was
|
||||
/// secure, the client address after RustFS's trusted-proxy correction, and RustFS's request
|
||||
/// identifier. Each is derived from what RustFS's own layers already attached; no header is read.
|
||||
fn attach_host_facts(extensions: &mut Extensions) {
|
||||
let client_info = extensions.get::<ClientInfo>();
|
||||
let native_tls = extensions.get::<NativeTlsTransport>().is_some_and(|transport| transport.0);
|
||||
let security = if is_secure_transport(native_tls, client_info) {
|
||||
TransportSecurity::Encrypted
|
||||
} else {
|
||||
TransportSecurity::Plaintext
|
||||
};
|
||||
let client_ip = client_info.map(|info| info.real_ip).or_else(|| {
|
||||
extensions
|
||||
.get::<Option<RemoteAddr>>()
|
||||
.and_then(|peer| peer.as_ref().map(|peer| peer.0.ip()))
|
||||
});
|
||||
// An identifier outside the gateway's alphabet is not handed over; the gateway mints its own.
|
||||
let request_id = extensions
|
||||
.get::<RequestContext>()
|
||||
.and_then(|context| HostRequestId::new(&context.request_id).ok());
|
||||
|
||||
extensions.insert(security);
|
||||
if let Some(client_ip) = client_ip {
|
||||
extensions.insert(ClientAddr::from_peer(client_ip));
|
||||
}
|
||||
if let Some(request_id) = request_id {
|
||||
extensions.insert(request_id);
|
||||
}
|
||||
}
|
||||
|
||||
/// The root credentials of one server as they are now, read on every call from where the legacy
|
||||
/// stack reads them, so a root change reaches both stacks at once. `None` while they are not
|
||||
/// initialized or cannot verify a signature, which the adapters answer as unavailable.
|
||||
fn live_root(server_ctx: Arc<ServerContextSlot>) -> impl Fn() -> Option<Arc<RootCredentials>> + Send + Sync + 'static {
|
||||
move || {
|
||||
let root = match server_ctx.app_context() {
|
||||
Some(context) => context.action_credentials().get(),
|
||||
None => current_action_credentials(),
|
||||
}?;
|
||||
// IAM signs STS session tokens with its token-signing key, which is the root secret
|
||||
// until IAM has published one of its own.
|
||||
let signing_key = get_token_signing_key().unwrap_or_else(|| root.secret_key.clone());
|
||||
RootCredentials::new(&root.access_key, &root.secret_key, &signing_key)
|
||||
.ok()
|
||||
.map(Arc::new)
|
||||
}
|
||||
}
|
||||
|
||||
/// The IAM system of one server, looked up when a credential is: IAM is installed after the
|
||||
/// listener starts, so the assembly cannot hold a handle.
|
||||
struct ServerIam {
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
}
|
||||
|
||||
impl ServerIam {
|
||||
fn ready(&self) -> Option<Arc<IamSys<ObjectStore>>> {
|
||||
self.server_ctx
|
||||
.app_context()
|
||||
.filter(|context| context.iam().is_ready())
|
||||
.map(|context| context.iam().handle())
|
||||
}
|
||||
}
|
||||
|
||||
impl IamKeyLookup for ServerIam {
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready().is_some()
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
|
||||
Box::pin(async move {
|
||||
let Some(iam) = self.ready() else {
|
||||
return Err(IamError::IamSysNotInitialized);
|
||||
};
|
||||
iam.check_key(access_key).await
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl IamPrincipalLookup for ServerIam {
|
||||
fn group_memberships(&self, user: &str) -> Vec<String> {
|
||||
self.ready().map(|iam| iam.user_group_memberships(user)).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -1,687 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! The gateway `Authorizer` of the RustFS assembly: an early gate, never the authorization.
|
||||
//!
|
||||
//! Responsible for: [`AUTHORIZATION_TABLE`], one row per registered operation, and
|
||||
//! [`RustfsAuthorizer`], which answers both gateway authorization stages from it.
|
||||
//! NOT responsible for: the authoritative decision. That is the legacy access hook in its owned
|
||||
//! form, run by the operation's handler immediately before its use case
|
||||
//! (`server/gateway_handlers`), where the legacy stack runs it; decisions, error codes and their
|
||||
//! order are therefore the same code on both stacks (rustfs/backlog#2761).
|
||||
//! Upstream: the gateway's route and input authorization stages. Downstream: the shared
|
||||
//! authorization core, through [`DecisionCore`].
|
||||
//!
|
||||
//! # What this gate may do
|
||||
//!
|
||||
//! It may refuse a request the hook would certainly refuse too, and nothing else. It is never
|
||||
//! the only check, and it never allows by default:
|
||||
//!
|
||||
//! - an operation without a row is denied;
|
||||
//! - a `RouteDecidable` row asks the shared decision core the hook asks, with the same facts;
|
||||
//! - a `DeferToHandlerHook` row evaluates only its [`RouteGate`], which names exactly what the
|
||||
//! route stage can know, and leaves the rest to the hook;
|
||||
//! - a `LegacyNoCheck` row is an operation whose legacy hook is empty, said so explicitly;
|
||||
//! - a core that cannot answer is `Indeterminate`, which the gateway refuses.
|
||||
//!
|
||||
//! Plain rows therefore evaluate policy twice, once here and once in the hook. That is accepted
|
||||
//! until it is measured.
|
||||
//!
|
||||
//! A row may be `RouteDecidable` only when its denial concerns the one routed resource and the
|
||||
//! one routed action. An operation whose denial can concern a second resource (a copy source) or
|
||||
//! a version-specific action (a versioned read) is a `DeferToHandlerHook` row: the hook writes
|
||||
//! the denial record with the resource and action it actually refused, and an early refusal
|
||||
//! here would record the wrong ones.
|
||||
//!
|
||||
//! # Deferred rows
|
||||
//!
|
||||
//! Every row that is not `RouteDecidable` is listed here, and the census below holds the table
|
||||
//! to these two lists, so a deferral cannot be added without this section changing.
|
||||
//!
|
||||
//! `DeferToHandlerHook`:
|
||||
//!
|
||||
//! - `ListBuckets`: the legacy hook is empty and the use case decides per caller. It lists every
|
||||
//! bucket for a caller allowed `s3:ListAllMyBuckets`, otherwise the buckets the caller may list
|
||||
//! or locate, and refuses when there is none; that needs the bucket inventory. The route stage
|
||||
//! refuses an anonymous caller, which the use case refuses with the same answer.
|
||||
//!
|
||||
//! `LegacyNoCheck`: none.
|
||||
|
||||
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead, authorize_owned_request};
|
||||
use crate::auth::get_session_token;
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::shared_types::RemoteAddr;
|
||||
use crate::storage_api::server::http::access::resolve_req_info;
|
||||
use crate::storage_api::server::http::request_context::RequestContext;
|
||||
use http::{Extensions, HeaderMap, Method, Uri};
|
||||
use rustfs_gateway::{
|
||||
Authorizer, AuthzRequest, BoxFuture, Decision, InputAuthzRequest, InputDecisions, RequestContext as GatewayRequestContext,
|
||||
};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The resource an operation's action is asked about.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ResourceForm {
|
||||
/// The service itself: no bucket and no key.
|
||||
Service,
|
||||
/// One bucket.
|
||||
#[cfg_attr(not(test), expect(dead_code, reason = "constructed by the first bucket-level row"))]
|
||||
Bucket,
|
||||
/// One object in one bucket.
|
||||
#[cfg_attr(not(test), expect(dead_code, reason = "constructed by the first object-level row"))]
|
||||
Object,
|
||||
}
|
||||
|
||||
/// What a deferred row's route stage evaluates: only facts that do not depend on the body.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum RouteGate {
|
||||
/// Refuses a request that presented no credentials. A request with a verified caller passes
|
||||
/// to the hook, which decides.
|
||||
AuthenticatedCaller,
|
||||
}
|
||||
|
||||
/// How much of an operation's decision the route stage can make.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum RouteStage {
|
||||
/// The route stage asks the shared decision core the same question, with the same facts, as
|
||||
/// the hook.
|
||||
#[cfg_attr(not(test), expect(dead_code, reason = "constructed by the first plain operation row"))]
|
||||
RouteDecidable,
|
||||
/// The route stage evaluates `gate` and the hook in the handler decides the rest.
|
||||
DeferToHandlerHook {
|
||||
/// Exactly what the route stage evaluates.
|
||||
gate: RouteGate,
|
||||
/// Why the route stage cannot decide the whole of it.
|
||||
reason: &'static str,
|
||||
},
|
||||
/// The legacy hook for this operation is empty, so there is nothing to evaluate early.
|
||||
#[cfg_attr(
|
||||
not(test),
|
||||
expect(dead_code, reason = "constructed by the row of an operation whose legacy hook is empty")
|
||||
)]
|
||||
LegacyNoCheck {
|
||||
/// Why legacy checks nothing.
|
||||
reason: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
/// The checks an operation's legacy hook performs beyond its one action, which the shared
|
||||
/// decision core does not perform. There is no default: every row states this column.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ExtraChecks {
|
||||
/// The hook asks its one action and nothing else.
|
||||
#[cfg_attr(not(test), expect(dead_code, reason = "constructed by the first plain operation row"))]
|
||||
None,
|
||||
/// The hook, or the use case it defers to, also performs these.
|
||||
Performed(&'static [&'static str]),
|
||||
}
|
||||
|
||||
/// One operation's authorization, as the legacy access hook performs it.
|
||||
///
|
||||
/// A struct literal names every column, so a row cannot leave one unspecified.
|
||||
#[cfg_attr(
|
||||
not(test),
|
||||
expect(
|
||||
dead_code,
|
||||
reason = "the body-input and extra-check columns record what the handler hook does and are read by the census"
|
||||
)
|
||||
)]
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) struct AuthorizationRow {
|
||||
/// The operation, by the name it is registered under.
|
||||
pub(crate) operation: &'static str,
|
||||
/// The action the legacy hook asks, mapped from the operation and not from the gateway's
|
||||
/// spelling of its action: the two differ for 22 operations, and legacy's is the baseline.
|
||||
pub(crate) action: Action,
|
||||
pub(crate) resource: ResourceForm,
|
||||
pub(crate) route_stage: RouteStage,
|
||||
/// The body-derived inputs the hook needs, which no route stage has.
|
||||
pub(crate) body_inputs: &'static [&'static str],
|
||||
pub(crate) extra_checks: ExtraChecks,
|
||||
}
|
||||
|
||||
/// Every operation the gateway stack registers. Kept equal to the registered set by the census
|
||||
/// in `server/gateway_stack.rs`.
|
||||
pub(crate) static AUTHORIZATION_TABLE: &[AuthorizationRow] = &[AuthorizationRow {
|
||||
operation: "ListBuckets",
|
||||
action: Action::S3Action(S3Action::ListAllMyBucketsAction),
|
||||
resource: ResourceForm::Service,
|
||||
route_stage: RouteStage::DeferToHandlerHook {
|
||||
gate: RouteGate::AuthenticatedCaller,
|
||||
reason: "the use case decides per caller from the bucket inventory",
|
||||
},
|
||||
body_inputs: &[],
|
||||
extra_checks: ExtraChecks::Performed(&[
|
||||
"a caller without s3:ListAllMyBuckets sees the buckets it may list (s3:ListBucket) or locate (s3:GetBucketLocation), \
|
||||
judged by IAM alone, and is refused when there is none",
|
||||
]),
|
||||
}];
|
||||
|
||||
/// The request line as it reached RustFS, attached by the host before the gateway accepts the
|
||||
/// request. The gateway hands an authorizer the headers and the query, but neither the method
|
||||
/// nor the target, and the shared core is asked about the request as it arrived.
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct RequestLine {
|
||||
pub(crate) method: Method,
|
||||
pub(crate) uri: Uri,
|
||||
}
|
||||
|
||||
/// The request facts the shared decision core is asked with, each copied from where the gateway
|
||||
/// or a RustFS layer in front of it put it.
|
||||
pub(crate) struct RouteFacts<'a> {
|
||||
/// The access key the gateway verified; `None` for an anonymous request.
|
||||
pub(crate) access_key: Option<&'a str>,
|
||||
/// The bucket, key and version the gateway routed, when the request names them.
|
||||
pub(crate) bucket: Option<&'a str>,
|
||||
pub(crate) key: Option<&'a str>,
|
||||
pub(crate) version_id: Option<&'a str>,
|
||||
/// The accepted header lines; `None` when the gateway handed over none.
|
||||
pub(crate) headers: Option<HeaderMap>,
|
||||
pub(crate) request_line: Option<&'a RequestLine>,
|
||||
pub(crate) client_info: Option<&'a ClientInfo>,
|
||||
pub(crate) remote_addr: Option<SocketAddr>,
|
||||
pub(crate) request_context: Option<&'a RequestContext>,
|
||||
}
|
||||
|
||||
/// The shared authorization decision core, as the route stage of a `RouteDecidable` row asks it.
|
||||
pub(crate) trait DecisionCore: Send + Sync + 'static {
|
||||
/// Decides `row`'s action for the caller and resource of `facts`: `Ok` to allow,
|
||||
/// `AccessDenied` to deny, and any other error when the core could not answer.
|
||||
fn decide<'a>(&'a self, row: &'a AuthorizationRow, facts: &'a RouteFacts<'a>) -> BoxFuture<'a, S3Result<()>>;
|
||||
}
|
||||
|
||||
/// The decision core both stacks share: the owned authorization entry point, asked about the
|
||||
/// request the route stage sees.
|
||||
///
|
||||
/// The caller is resolved by the function the legacy access check resolves it with, and the
|
||||
/// decision is `authorize_owned_request`, which the handler's access hook calls too. It is only
|
||||
/// the policy decision: a row that needs a body-derived input or a composite check is not
|
||||
/// `RouteDecidable`.
|
||||
pub(crate) struct OwnedDecisionCore {
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
}
|
||||
|
||||
impl OwnedDecisionCore {
|
||||
pub(crate) fn new(server_ctx: Arc<ServerContextSlot>) -> Self {
|
||||
Self { server_ctx }
|
||||
}
|
||||
|
||||
async fn authorize(&self, row: &AuthorizationRow, facts: &RouteFacts<'_>) -> S3Result<()> {
|
||||
let (Some(line), Some(headers)) = (facts.request_line, facts.headers.as_ref()) else {
|
||||
return Err(unanswerable("the route stage was handed no request line or headers"));
|
||||
};
|
||||
let session_token = get_session_token(&line.uri, headers).unwrap_or_default().to_owned();
|
||||
let mut req_info =
|
||||
resolve_req_info(&self.server_ctx, facts.access_key, &session_token, facts.request_context.cloned()).await?;
|
||||
match row.resource {
|
||||
ResourceForm::Service => {}
|
||||
ResourceForm::Bucket => req_info.bucket = Some(routed(facts.bucket)?),
|
||||
ResourceForm::Object => {
|
||||
req_info.bucket = Some(routed(facts.bucket)?);
|
||||
req_info.object = Some(routed(facts.key)?);
|
||||
req_info.version_id = facts.version_id.map(str::to_owned);
|
||||
}
|
||||
}
|
||||
|
||||
let credentials = facts.access_key.map(|access_key| {
|
||||
let session_token = (!session_token.is_empty()).then(|| session_token.clone());
|
||||
RequestCredentials::without_secret_key(access_key.to_owned(), session_token)
|
||||
});
|
||||
let mut extensions = Extensions::new();
|
||||
extensions.insert(facts.remote_addr.map(RemoteAddr));
|
||||
if let Some(client_info) = facts.client_info {
|
||||
extensions.insert(client_info.clone());
|
||||
}
|
||||
if let Some(request_context) = facts.request_context {
|
||||
extensions.insert(request_context.clone());
|
||||
}
|
||||
extensions.insert(Arc::clone(&self.server_ctx));
|
||||
let head = RequestHead {
|
||||
method: line.method.clone(),
|
||||
uri: line.uri.clone(),
|
||||
headers: headers.clone(),
|
||||
credentials,
|
||||
region: None,
|
||||
service: None,
|
||||
};
|
||||
let mut envelope = RequestEnvelope::from_parts(head, extensions);
|
||||
envelope.install_req_info(req_info);
|
||||
authorize_owned_request(&mut envelope, row.action, None).await
|
||||
}
|
||||
}
|
||||
|
||||
impl DecisionCore for OwnedDecisionCore {
|
||||
fn decide<'a>(&'a self, row: &'a AuthorizationRow, facts: &'a RouteFacts<'a>) -> BoxFuture<'a, S3Result<()>> {
|
||||
Box::pin(self.authorize(row, facts))
|
||||
}
|
||||
}
|
||||
|
||||
/// The routed name a row's resource form requires. Its absence is a question the core cannot
|
||||
/// answer, never a decision about some other resource.
|
||||
fn routed(name: Option<&str>) -> S3Result<String> {
|
||||
name.map(str::to_owned)
|
||||
.ok_or_else(|| unanswerable("the route stage was handed no name for the row's resource"))
|
||||
}
|
||||
|
||||
fn unanswerable(message: &'static str) -> S3Error {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, message)
|
||||
}
|
||||
|
||||
/// What the table decides about one request by itself, and what it leaves to the core.
|
||||
enum Question<'a> {
|
||||
Decided(Decision),
|
||||
AskCore(&'static AuthorizationRow, Box<RouteFacts<'a>>),
|
||||
}
|
||||
|
||||
/// The RustFS implementation of the gateway's two authorization stages.
|
||||
pub(crate) struct RustfsAuthorizer {
|
||||
table: &'static [AuthorizationRow],
|
||||
core: Arc<dyn DecisionCore>,
|
||||
}
|
||||
|
||||
impl RustfsAuthorizer {
|
||||
/// The authorizer over `table`, asking `core` for `RouteDecidable` rows.
|
||||
pub(crate) fn new(table: &'static [AuthorizationRow], core: Arc<dyn DecisionCore>) -> Self {
|
||||
Self { table, core }
|
||||
}
|
||||
|
||||
/// Reads `operation`'s row. `facts` is built only for a row that asks the core.
|
||||
fn question<'f>(&self, operation: &str, anonymous: bool, facts: impl FnOnce() -> RouteFacts<'f>) -> Question<'f> {
|
||||
let table: &'static [AuthorizationRow] = self.table;
|
||||
let Some(row) = table.iter().find(|row| row.operation == operation) else {
|
||||
// No row, no answer to give: refused, never allowed.
|
||||
return Question::Decided(Decision::Deny);
|
||||
};
|
||||
match row.route_stage {
|
||||
RouteStage::RouteDecidable => Question::AskCore(row, Box::new(facts())),
|
||||
RouteStage::DeferToHandlerHook { gate, .. } => Question::Decided(match gate {
|
||||
RouteGate::AuthenticatedCaller if anonymous => Decision::Deny,
|
||||
RouteGate::AuthenticatedCaller => Decision::Allow,
|
||||
}),
|
||||
RouteStage::LegacyNoCheck { .. } => Question::Decided(Decision::Allow),
|
||||
}
|
||||
}
|
||||
|
||||
async fn answer(&self, question: Question<'_>) -> Decision {
|
||||
match question {
|
||||
Question::Decided(decision) => decision,
|
||||
Question::AskCore(row, facts) => core_decision(self.core.decide(row, &facts).await),
|
||||
}
|
||||
}
|
||||
|
||||
fn ask<'a>(&self, context: &GatewayRequestContext<'a>, request: &AuthzRequest<'a>) -> Question<'a> {
|
||||
self.question(request.operation, request.identity.is_none(), || route_facts(context, request))
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads the core's answer as a decision. Only an explicit denial is a `Deny`; a failure to
|
||||
/// answer is `Indeterminate` and never an allowance.
|
||||
fn core_decision(answer: S3Result<()>) -> Decision {
|
||||
match answer {
|
||||
Ok(()) => Decision::Allow,
|
||||
Err(error) if error.code() == &S3ErrorCode::AccessDenied => Decision::Deny,
|
||||
Err(_) => Decision::Indeterminate,
|
||||
}
|
||||
}
|
||||
|
||||
fn route_facts<'a>(context: &GatewayRequestContext<'a>, request: &AuthzRequest<'a>) -> RouteFacts<'a> {
|
||||
let host = context.server_extensions();
|
||||
RouteFacts {
|
||||
access_key: request.identity.map(|identity| identity.access_key_id()),
|
||||
bucket: request.bucket.map(|bucket| bucket.as_str()),
|
||||
key: request.key.map(|key| key.as_str()),
|
||||
version_id: request.version_id,
|
||||
headers: context.headers().map(|accepted| {
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in accepted.iter_raw() {
|
||||
headers.append(name.clone(), value.clone());
|
||||
}
|
||||
headers
|
||||
}),
|
||||
request_line: host.get::<RequestLine>(),
|
||||
client_info: host.get::<ClientInfo>(),
|
||||
remote_addr: host
|
||||
.get::<Option<RemoteAddr>>()
|
||||
.and_then(|peer| peer.as_ref().map(|peer| peer.0)),
|
||||
request_context: host.get::<RequestContext>(),
|
||||
}
|
||||
}
|
||||
|
||||
impl Authorizer for RustfsAuthorizer {
|
||||
fn authorize_route<'a>(
|
||||
&'a self,
|
||||
context: &'a GatewayRequestContext<'a>,
|
||||
request: &'a AuthzRequest<'a>,
|
||||
) -> BoxFuture<'a, Decision> {
|
||||
let question = self.ask(context, request);
|
||||
Box::pin(self.answer(question))
|
||||
}
|
||||
|
||||
fn authorize_input<'a>(
|
||||
&'a self,
|
||||
context: &'a GatewayRequestContext<'a>,
|
||||
request: &'a InputAuthzRequest<'a>,
|
||||
) -> BoxFuture<'a, InputDecisions> {
|
||||
// `decide_all` visits every derived resource and then the visibility question, in that
|
||||
// order; each is read from the table now and answered below, where the core is awaited.
|
||||
let stage = self.ask(context, request.route());
|
||||
let mut derived: Vec<Question<'a>> = request
|
||||
.resources()
|
||||
.iter()
|
||||
.map(|resource| self.ask(context, resource))
|
||||
.collect();
|
||||
if let Some(visibility) = request.visibility() {
|
||||
derived.push(self.ask(context, visibility));
|
||||
}
|
||||
Box::pin(async move {
|
||||
let stage = self.answer(stage).await;
|
||||
let mut answers = Vec::with_capacity(derived.len());
|
||||
for question in derived {
|
||||
answers.push(self.answer(question).await);
|
||||
}
|
||||
let mut answers = answers.into_iter();
|
||||
request.decide_all(stage, |_| answers.next().unwrap_or(Decision::Deny))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Mutex;
|
||||
|
||||
/// The rows documented as `DeferToHandlerHook` in this file's header.
|
||||
const DOCUMENTED_DEFERRED_ROWS: &[&str] = &["ListBuckets"];
|
||||
/// The rows documented as `LegacyNoCheck` in this file's header.
|
||||
const DOCUMENTED_NO_CHECK_ROWS: &[&str] = &[];
|
||||
|
||||
static PLAIN_TABLE: &[AuthorizationRow] = &[
|
||||
AuthorizationRow {
|
||||
operation: "HeadBucket",
|
||||
action: Action::S3Action(S3Action::ListBucketAction),
|
||||
resource: ResourceForm::Bucket,
|
||||
route_stage: RouteStage::RouteDecidable,
|
||||
body_inputs: &[],
|
||||
extra_checks: ExtraChecks::None,
|
||||
},
|
||||
AuthorizationRow {
|
||||
operation: "GetObjectTorrent",
|
||||
action: Action::S3Action(S3Action::GetObjectAction),
|
||||
resource: ResourceForm::Object,
|
||||
route_stage: RouteStage::LegacyNoCheck {
|
||||
reason: "the legacy hook is empty",
|
||||
},
|
||||
body_inputs: &[],
|
||||
extra_checks: ExtraChecks::None,
|
||||
},
|
||||
];
|
||||
|
||||
/// A core that gives one scripted answer and records what it was asked.
|
||||
struct ScriptedCore {
|
||||
answer: fn() -> S3Result<()>,
|
||||
asked: Mutex<Vec<(&'static str, Action, Option<String>)>>,
|
||||
}
|
||||
|
||||
impl ScriptedCore {
|
||||
fn answering(answer: fn() -> S3Result<()>) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
answer,
|
||||
asked: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl DecisionCore for ScriptedCore {
|
||||
fn decide<'a>(&'a self, row: &'a AuthorizationRow, facts: &'a RouteFacts<'a>) -> BoxFuture<'a, S3Result<()>> {
|
||||
self.asked
|
||||
.lock()
|
||||
.expect("core lock")
|
||||
.push((row.operation, row.action, facts.access_key.map(str::to_owned)));
|
||||
let answer = (self.answer)();
|
||||
Box::pin(async move { answer })
|
||||
}
|
||||
}
|
||||
|
||||
fn denied() -> S3Result<()> {
|
||||
Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"))
|
||||
}
|
||||
|
||||
fn unavailable() -> S3Result<()> {
|
||||
Err(S3Error::with_message(S3ErrorCode::InternalError, "the policy store is unavailable"))
|
||||
}
|
||||
|
||||
/// One request as a test asks about it.
|
||||
struct Asked<'a> {
|
||||
operation: &'a str,
|
||||
facts: RouteFacts<'a>,
|
||||
}
|
||||
|
||||
fn facts<'a>(operation: &'a str, access_key: Option<&'a str>) -> Asked<'a> {
|
||||
Asked {
|
||||
operation,
|
||||
facts: RouteFacts {
|
||||
access_key,
|
||||
bucket: None,
|
||||
key: None,
|
||||
version_id: None,
|
||||
headers: None,
|
||||
request_line: None,
|
||||
client_info: None,
|
||||
remote_addr: None,
|
||||
request_context: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn decision(table: &'static [AuthorizationRow], core: Arc<dyn DecisionCore>, asked: Asked<'_>) -> Decision {
|
||||
let authorizer = RustfsAuthorizer::new(table, core);
|
||||
let anonymous = asked.facts.access_key.is_none();
|
||||
let question = authorizer.question(asked.operation, anonymous, || asked.facts);
|
||||
authorizer.answer(question).await
|
||||
}
|
||||
|
||||
/// The real decision core of a server that has no object store yet.
|
||||
fn owned_core() -> Arc<dyn DecisionCore> {
|
||||
Arc::new(OwnedDecisionCore::new(ServerContextSlot::new()))
|
||||
}
|
||||
|
||||
fn head_bucket() -> RequestLine {
|
||||
RequestLine {
|
||||
method: Method::HEAD,
|
||||
uri: Uri::from_static("/bucket"),
|
||||
}
|
||||
}
|
||||
|
||||
fn rows_where(predicate: fn(&RouteStage) -> bool) -> BTreeSet<&'static str> {
|
||||
AUTHORIZATION_TABLE
|
||||
.iter()
|
||||
.filter(|row| predicate(&row.route_stage))
|
||||
.map(|row| row.operation)
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_operation_without_a_row_is_denied() {
|
||||
let core = ScriptedCore::answering(|| Ok(()));
|
||||
|
||||
for operation in ["GetObject", "PutObject", "", "listbuckets"] {
|
||||
let decided = decision(AUTHORIZATION_TABLE, core.clone(), facts(operation, Some("AKIDUSER0001"))).await;
|
||||
assert_eq!(decided, Decision::Deny, "{operation:?}");
|
||||
}
|
||||
assert!(core.asked.lock().expect("core lock").is_empty(), "an unmapped operation asks nothing");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_core_that_cannot_answer_is_indeterminate_never_an_allowance() {
|
||||
let decided = decision(
|
||||
PLAIN_TABLE,
|
||||
ScriptedCore::answering(unavailable),
|
||||
facts("HeadBucket", Some("AKIDUSER0001")),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(decided, Decision::Indeterminate);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_owned_core_without_a_store_cannot_answer_and_never_allows() {
|
||||
let line = head_bucket();
|
||||
let mut asked = facts("HeadBucket", None);
|
||||
asked.facts.bucket = Some("bucket");
|
||||
asked.facts.headers = Some(HeaderMap::new());
|
||||
asked.facts.request_line = Some(&line);
|
||||
|
||||
assert_eq!(decision(PLAIN_TABLE, owned_core(), asked).await, Decision::Indeterminate);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_owned_core_refuses_a_request_the_host_did_not_describe() {
|
||||
let line = head_bucket();
|
||||
|
||||
let mut without_a_line = facts("HeadBucket", None);
|
||||
without_a_line.facts.bucket = Some("bucket");
|
||||
without_a_line.facts.headers = Some(HeaderMap::new());
|
||||
assert_eq!(decision(PLAIN_TABLE, owned_core(), without_a_line).await, Decision::Indeterminate);
|
||||
|
||||
let mut without_headers = facts("HeadBucket", None);
|
||||
without_headers.facts.bucket = Some("bucket");
|
||||
without_headers.facts.request_line = Some(&line);
|
||||
assert_eq!(decision(PLAIN_TABLE, owned_core(), without_headers).await, Decision::Indeterminate);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_owned_core_refuses_a_bucket_row_whose_request_names_no_bucket() {
|
||||
let line = head_bucket();
|
||||
let mut asked = facts("HeadBucket", None);
|
||||
asked.facts.headers = Some(HeaderMap::new());
|
||||
asked.facts.request_line = Some(&line);
|
||||
|
||||
assert_eq!(decision(PLAIN_TABLE, owned_core(), asked).await, Decision::Indeterminate);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_core_denial_is_a_denial() {
|
||||
let decided = decision(PLAIN_TABLE, ScriptedCore::answering(denied), facts("HeadBucket", Some("AKIDUSER0001"))).await;
|
||||
|
||||
assert_eq!(decided, Decision::Deny);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_route_decidable_row_asks_the_core_its_legacy_action_for_the_verified_caller() {
|
||||
let core = ScriptedCore::answering(|| Ok(()));
|
||||
|
||||
let decided = decision(PLAIN_TABLE, core.clone(), facts("HeadBucket", Some("AKIDUSER0001"))).await;
|
||||
|
||||
assert_eq!(decided, Decision::Allow);
|
||||
assert_eq!(
|
||||
*core.asked.lock().expect("core lock"),
|
||||
[(
|
||||
"HeadBucket",
|
||||
Action::S3Action(S3Action::ListBucketAction),
|
||||
Some("AKIDUSER0001".to_owned())
|
||||
)]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_deferred_row_refuses_an_anonymous_caller_without_asking_the_core() {
|
||||
let core = ScriptedCore::answering(|| Ok(()));
|
||||
|
||||
assert_eq!(
|
||||
decision(AUTHORIZATION_TABLE, core.clone(), facts("ListBuckets", None)).await,
|
||||
Decision::Deny
|
||||
);
|
||||
assert_eq!(
|
||||
decision(AUTHORIZATION_TABLE, core.clone(), facts("ListBuckets", Some("AKIDUSER0001"))).await,
|
||||
Decision::Allow,
|
||||
"a verified caller passes to the hook"
|
||||
);
|
||||
assert!(
|
||||
core.asked.lock().expect("core lock").is_empty(),
|
||||
"a deferred row decides nothing through the core"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_no_check_row_is_allowed_only_because_its_row_says_so() {
|
||||
let core = ScriptedCore::answering(denied);
|
||||
|
||||
assert_eq!(
|
||||
decision(PLAIN_TABLE, core.clone(), facts("GetObjectTorrent", None)).await,
|
||||
Decision::Allow
|
||||
);
|
||||
assert_eq!(decision(AUTHORIZATION_TABLE, core, facts("GetObjectTorrent", None)).await, Decision::Deny);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_deferred_rows_are_exactly_the_documented_ones() {
|
||||
let deferred = rows_where(|stage| matches!(stage, RouteStage::DeferToHandlerHook { .. }));
|
||||
|
||||
assert_eq!(deferred, DOCUMENTED_DEFERRED_ROWS.iter().copied().collect::<BTreeSet<_>>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_no_check_rows_are_exactly_the_documented_ones() {
|
||||
let unchecked = rows_where(|stage| matches!(stage, RouteStage::LegacyNoCheck { .. }));
|
||||
|
||||
assert_eq!(unchecked, DOCUMENTED_NO_CHECK_ROWS.iter().copied().collect::<BTreeSet<_>>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_row_states_every_column_and_names_one_operation() {
|
||||
let mut seen = BTreeSet::new();
|
||||
for row in AUTHORIZATION_TABLE {
|
||||
assert!(seen.insert(row.operation), "{} has two rows", row.operation);
|
||||
assert!(!row.operation.is_empty());
|
||||
match row.route_stage {
|
||||
RouteStage::DeferToHandlerHook { reason, .. } | RouteStage::LegacyNoCheck { reason } => {
|
||||
assert!(!reason.is_empty(), "{} defers without a reason", row.operation);
|
||||
}
|
||||
RouteStage::RouteDecidable => {}
|
||||
}
|
||||
match row.extra_checks {
|
||||
ExtraChecks::Performed(checks) => {
|
||||
assert!(!checks.is_empty(), "{} lists no extra check; say `ExtraChecks::None`", row.operation);
|
||||
assert!(checks.iter().all(|check| !check.is_empty()), "{}", row.operation);
|
||||
}
|
||||
ExtraChecks::None => {}
|
||||
}
|
||||
assert!(row.body_inputs.iter().all(|input| !input.is_empty()), "{}", row.operation);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_buckets_asks_the_legacy_action_on_the_service() {
|
||||
let row = AUTHORIZATION_TABLE
|
||||
.iter()
|
||||
.find(|row| row.operation == "ListBuckets")
|
||||
.expect("ListBuckets is registered");
|
||||
|
||||
assert_eq!(row.action, Action::S3Action(S3Action::ListAllMyBucketsAction));
|
||||
assert_eq!(row.resource, ResourceForm::Service);
|
||||
assert_eq!(
|
||||
row.route_stage,
|
||||
RouteStage::DeferToHandlerHook {
|
||||
gate: RouteGate::AuthenticatedCaller,
|
||||
reason: "the use case decides per caller from the bucket inventory",
|
||||
}
|
||||
);
|
||||
assert!(row.body_inputs.is_empty());
|
||||
assert!(matches!(row.extra_checks, ExtraChecks::Performed(_)));
|
||||
}
|
||||
}
|
||||
@@ -1,561 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! The assembled gateway stack, driven through [`GatewayStack::call`] exactly as `server/http.rs`
|
||||
//! drives it. Requests that need object storage are in `rustfs/tests/embedded_gateway_stack_test.rs`.
|
||||
|
||||
use super::authorizer::AUTHORIZATION_TABLE;
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, Method, StatusCode};
|
||||
use http_body_util::{BodyExt, Empty};
|
||||
use rustfs_gateway::{AuthzRequest, ClientFacts, Decision, InputAuthzRequest, InputDecisions, NO_DEADLINE};
|
||||
use std::collections::BTreeSet;
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Mutex;
|
||||
use std::time::Instant;
|
||||
|
||||
const ROOT_ACCESS_KEY: &str = "rustfsgatewayroot";
|
||||
const ROOT_SECRET_KEY: &str = "rustfs-gateway-root-secret";
|
||||
const REQUEST_ID: &str = "7c9e6679-7425-40de-944b-e07fc1f90ae7";
|
||||
const PEER: ([u8; 4], u16) = ([10, 0, 0, 7], 41000);
|
||||
const CLIENT: [u8; 4] = [203, 0, 113, 9];
|
||||
const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
||||
|
||||
fn assembly() -> GatewayAssembly {
|
||||
GatewayAssembly {
|
||||
server_ctx: ServerContextSlot::new(),
|
||||
root_access_key: ROOT_ACCESS_KEY.to_owned(),
|
||||
root_secret_key: ROOT_SECRET_KEY.to_owned(),
|
||||
region: Some("us-east-1".to_owned()),
|
||||
server_domains: Vec::new(),
|
||||
body_read_timeout: Duration::from_secs(300),
|
||||
keystone_enabled: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// An IAM whose readiness and answer are scripted.
|
||||
struct ScriptedIam {
|
||||
ready: bool,
|
||||
}
|
||||
|
||||
impl IamKeyLookup for ScriptedIam {
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
|
||||
Box::pin(async { Ok((None, false)) })
|
||||
}
|
||||
}
|
||||
|
||||
impl IamPrincipalLookup for ScriptedIam {
|
||||
fn group_memberships(&self, _user: &str) -> Vec<String> {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// A settable live root, standing in for the server's runtime credentials.
|
||||
#[derive(Clone)]
|
||||
struct LiveRoot(Arc<Mutex<Option<Arc<RootCredentials>>>>);
|
||||
|
||||
impl LiveRoot {
|
||||
fn set(&self, root: RootCredentials) {
|
||||
*self.0.lock().expect("root lock") = Some(Arc::new(root));
|
||||
}
|
||||
|
||||
fn clear(&self) {
|
||||
*self.0.lock().expect("root lock") = None;
|
||||
}
|
||||
|
||||
fn source(&self) -> Arc<dyn RootSource> {
|
||||
let cell = Arc::clone(&self.0);
|
||||
Arc::new(move || cell.lock().expect("root lock").clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// A live root holding the test root.
|
||||
fn root() -> LiveRoot {
|
||||
let root = RootCredentials::new(ROOT_ACCESS_KEY, ROOT_SECRET_KEY, ROOT_SECRET_KEY).expect("a usable root");
|
||||
LiveRoot(Arc::new(Mutex::new(Some(Arc::new(root)))))
|
||||
}
|
||||
|
||||
/// The authorizer production installs: the registered table over the shared decision core.
|
||||
fn production_authorizer() -> RustfsAuthorizer {
|
||||
RustfsAuthorizer::new(AUTHORIZATION_TABLE, Arc::new(OwnedDecisionCore::new(ServerContextSlot::new())))
|
||||
}
|
||||
|
||||
/// The stack as production assembles it, over an IAM that knows no key but the root's.
|
||||
fn stack_over(iam: ScriptedIam) -> GatewayStack {
|
||||
let authorizer = production_authorizer();
|
||||
GatewayStack::assemble_with(assembly(), Arc::new(iam), root().source(), authorizer).expect("the stack assembles")
|
||||
}
|
||||
|
||||
fn request(method: Method, target: &str) -> http::request::Builder {
|
||||
Request::builder().method(method).uri(target).header("host", "localhost")
|
||||
}
|
||||
|
||||
/// A well-formed SigV4 header signature for `access_key` whose signature is not the right one.
|
||||
fn signed_with_a_wrong_signature(builder: http::request::Builder, access_key: &str) -> http::request::Builder {
|
||||
let now = jiff::Timestamp::now().strftime("%Y%m%dT%H%M%SZ").to_string();
|
||||
let date = &now[..8];
|
||||
builder
|
||||
.header("x-amz-date", now.as_str())
|
||||
.header("x-amz-content-sha256", EMPTY_SHA256)
|
||||
.header(
|
||||
"authorization",
|
||||
format!(
|
||||
"AWS4-HMAC-SHA256 Credential={access_key}/{date}/us-east-1/s3/aws4_request, \
|
||||
SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature={}",
|
||||
"0".repeat(64)
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
async fn answer(stack: &GatewayStack, builder: http::request::Builder) -> (StatusCode, HeaderMap, String) {
|
||||
let response = stack.call(builder.body(Empty::<Bytes>::new()).expect("static request")).await;
|
||||
let (parts, body) = response.into_parts();
|
||||
let body = body.collect().await.expect("the answer is read").to_bytes();
|
||||
(parts.status, parts.headers, String::from_utf8_lossy(&body).into_owned())
|
||||
}
|
||||
|
||||
fn rustfs_request_context(request_id: &str) -> RequestContext {
|
||||
RequestContext {
|
||||
request_id: request_id.to_owned(),
|
||||
x_amz_request_id: request_id.to_owned(),
|
||||
trace_id: None,
|
||||
span_id: None,
|
||||
start_time: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_registered_operation_has_an_authorization_row_and_every_row_an_operation() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let registered: BTreeSet<&str> = stack.registered_operations().into_iter().collect();
|
||||
let mapped: BTreeSet<&str> = AUTHORIZATION_TABLE.iter().map(|row| row.operation).collect();
|
||||
|
||||
assert_eq!(registered, mapped);
|
||||
assert_eq!(registered, BTreeSet::from(["ListBuckets"]), "the walking skeleton serves one operation");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn production_assembly_registers_what_the_test_assembly_does() {
|
||||
let stack = GatewayStack::assemble(assembly()).expect("the stack assembles");
|
||||
|
||||
assert_eq!(stack.registered_operations(), ["ListBuckets"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unusable_region_fails_the_assembly() {
|
||||
let refused = GatewayStack::assemble(GatewayAssembly {
|
||||
region: Some("Not A Region".to_owned()),
|
||||
..assembly()
|
||||
});
|
||||
|
||||
assert!(matches!(refused, Err(GatewayAssemblyError::Region { .. })), "{:?}", refused.err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unusable_root_access_key_fails_the_assembly_without_naming_the_secret() {
|
||||
let refused = GatewayStack::assemble(GatewayAssembly {
|
||||
root_access_key: String::new(),
|
||||
..assembly()
|
||||
});
|
||||
|
||||
let error = refused.err().expect("an empty access key verifies nothing");
|
||||
assert!(
|
||||
matches!(error, GatewayAssemblyError::RootCredentials(RootCredentialsError::InvalidAccessKey)),
|
||||
"{error:?}"
|
||||
);
|
||||
for rendered in [error.to_string(), format!("{error:?}")] {
|
||||
assert!(!rendered.contains(ROOT_SECRET_KEY), "{rendered}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_or_short_root_secret_fails_the_assembly() {
|
||||
for weak in ["", "1234567"] {
|
||||
let refused = GatewayStack::assemble(GatewayAssembly {
|
||||
root_secret_key: weak.to_owned(),
|
||||
..assembly()
|
||||
});
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
refused.as_ref().err(),
|
||||
Some(GatewayAssemblyError::RootCredentials(RootCredentialsError::WeakSecretKey))
|
||||
),
|
||||
"{} bytes",
|
||||
weak.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_root_change_reaches_the_next_request_and_a_withdrawn_root_signs_nothing() {
|
||||
const ROTATED_ACCESS_KEY: &str = "rotatedgatewayroot";
|
||||
let live = root();
|
||||
let authorizer = production_authorizer();
|
||||
let stack = GatewayStack::assemble_with(assembly(), Arc::new(ScriptedIam { ready: true }), live.source(), authorizer)
|
||||
.expect("the stack assembles");
|
||||
let signed_as = |access_key: &str| signed_with_a_wrong_signature(request(Method::GET, "/"), access_key);
|
||||
|
||||
let (_, _, known) = answer(&stack, signed_as(ROOT_ACCESS_KEY)).await;
|
||||
assert!(known.contains("<Code>SignatureDoesNotMatch</Code>"), "the root key is known: {known}");
|
||||
|
||||
// The root access key changes: the old one is now just an unknown key, and the new one is
|
||||
// known to the very next request.
|
||||
live.set(RootCredentials::new(ROTATED_ACCESS_KEY, ROOT_SECRET_KEY, ROOT_SECRET_KEY).expect("a usable root"));
|
||||
let (status, _, retired) = answer(&stack, signed_as(ROOT_ACCESS_KEY)).await;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "{retired}");
|
||||
assert!(retired.contains("<Code>InvalidAccessKeyId</Code>"), "{retired}");
|
||||
let (_, _, rotated) = answer(&stack, signed_as(ROTATED_ACCESS_KEY)).await;
|
||||
assert!(
|
||||
rotated.contains("<Code>SignatureDoesNotMatch</Code>"),
|
||||
"the new root key is known: {rotated}"
|
||||
);
|
||||
|
||||
// Injected: a root source that returns nothing, so the provider returns an error for the
|
||||
// lookup. The key that was root a moment ago verifies nothing; the pinned gateway answers the
|
||||
// provider fault as an unknown key. Ledger entry R-068 (rustfs/backlog#2684): this status
|
||||
// becomes `500` when the pin includes rustfs/gateway#1585 and the assembly sets its switch.
|
||||
live.clear();
|
||||
let (status, _, withdrawn) = answer(&stack, signed_as(ROTATED_ACCESS_KEY)).await;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "{withdrawn}");
|
||||
assert!(withdrawn.contains("<Code>InvalidAccessKeyId</Code>"), "{withdrawn}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keystone_enabled_fails_the_assembly_and_names_both_settings() {
|
||||
let refused = GatewayStack::assemble(GatewayAssembly {
|
||||
keystone_enabled: true,
|
||||
..assembly()
|
||||
});
|
||||
|
||||
let error = refused.err().expect("the gateway stack is not served with Keystone enabled");
|
||||
assert!(matches!(error, GatewayAssemblyError::KeystoneEnabled), "{error:?}");
|
||||
let message = error.to_string();
|
||||
for named in ["RUSTFS_S3_STACK=gateway", "RUSTFS_KEYSTONE_ENABLE", "rustfs/gateway#1586"] {
|
||||
assert!(message.contains(named), "{named} is missing from: {message}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keystone_disabled_assembles() {
|
||||
GatewayStack::assemble(GatewayAssembly {
|
||||
keystone_enabled: false,
|
||||
..assembly()
|
||||
})
|
||||
.expect("the gateway stack assembles without Keystone");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keystone_is_read_from_configuration_and_an_unreadable_one_counts_as_enabled() {
|
||||
const ENABLE: &str = "RUSTFS_KEYSTONE_ENABLE";
|
||||
const AUTH_URL: &str = "RUSTFS_KEYSTONE_AUTH_URL";
|
||||
|
||||
temp_env::with_vars([(ENABLE, None::<&str>), (AUTH_URL, None)], || {
|
||||
assert!(!keystone_enabled_in_configuration(), "unset is off");
|
||||
});
|
||||
temp_env::with_vars([(ENABLE, Some("false")), (AUTH_URL, None)], || {
|
||||
assert!(!keystone_enabled_in_configuration(), "false is off");
|
||||
});
|
||||
temp_env::with_vars([(ENABLE, Some("true")), (AUTH_URL, Some("http://keystone.example.test:5000"))], || {
|
||||
assert!(keystone_enabled_in_configuration(), "true is on");
|
||||
});
|
||||
temp_env::with_vars([(ENABLE, Some("true")), (AUTH_URL, None)], || {
|
||||
assert!(keystone_enabled_in_configuration(), "on but unreadable is still on");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_handler_runs_under_a_total_deadline_and_the_body_keeps_the_rustfs_idle_bound() {
|
||||
let config = service_config(Duration::from_secs(300));
|
||||
|
||||
assert_eq!(config.handler_deadline(HandlerDeadlineClass::Standard), NO_DEADLINE);
|
||||
assert_eq!(config.handler_deadline(HandlerDeadlineClass::Extended), NO_DEADLINE);
|
||||
assert_eq!(config.commit_progress_deadline(), NO_DEADLINE);
|
||||
assert_eq!(config.request_body_deadlines().read_idle(), Duration::from_secs(300));
|
||||
assert_eq!(config.request_body_deadlines().first_byte(), Duration::from_secs(300));
|
||||
assert_eq!(config.request_body_deadlines().throughput_window(), NO_DEADLINE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_disabled_body_read_timeout_lifts_the_idle_bound_instead_of_arming_zero() {
|
||||
let deadlines = body_deadlines(Duration::ZERO);
|
||||
|
||||
assert_eq!(deadlines.read_idle(), NO_DEADLINE);
|
||||
assert_eq!(deadlines.first_byte(), NO_DEADLINE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_facts_come_from_what_rustfs_layers_attached() {
|
||||
let peer = SocketAddr::from(PEER);
|
||||
let mut extensions = Extensions::new();
|
||||
extensions.insert(Some(RemoteAddr(peer)));
|
||||
extensions.insert(ClientInfo::direct(SocketAddr::new(IpAddr::from(CLIENT), 443)));
|
||||
extensions.insert(NativeTlsTransport(true));
|
||||
extensions.insert(rustfs_request_context(REQUEST_ID));
|
||||
|
||||
attach_host_facts(&mut extensions);
|
||||
|
||||
assert_eq!(extensions.get::<TransportSecurity>(), Some(&TransportSecurity::Encrypted));
|
||||
assert_eq!(
|
||||
extensions.get::<ClientAddr>().map(|address| address.ip()),
|
||||
Some(IpAddr::from(CLIENT)),
|
||||
"the trusted-proxy-corrected address wins over the socket peer"
|
||||
);
|
||||
assert_eq!(
|
||||
extensions
|
||||
.get::<HostRequestId>()
|
||||
.map(|id| id.request_id().as_str().to_owned()),
|
||||
Some(REQUEST_ID.to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_facts_never_lean_towards_secure_or_invent_an_address() {
|
||||
let mut bare = Extensions::new();
|
||||
attach_host_facts(&mut bare);
|
||||
assert_eq!(bare.get::<TransportSecurity>(), Some(&TransportSecurity::Plaintext));
|
||||
assert!(bare.get::<ClientAddr>().is_none(), "no address is known");
|
||||
assert!(bare.get::<HostRequestId>().is_none(), "no identifier is known");
|
||||
|
||||
let mut plaintext = Extensions::new();
|
||||
plaintext.insert(Some(RemoteAddr(SocketAddr::from(PEER))));
|
||||
plaintext.insert(NativeTlsTransport(false));
|
||||
plaintext.insert(rustfs_request_context("not a request id"));
|
||||
attach_host_facts(&mut plaintext);
|
||||
assert_eq!(plaintext.get::<TransportSecurity>(), Some(&TransportSecurity::Plaintext));
|
||||
assert_eq!(
|
||||
plaintext.get::<ClientAddr>().map(|address| address.ip()),
|
||||
Some(IpAddr::from(PEER.0)),
|
||||
"without a proxy the socket peer is the client"
|
||||
);
|
||||
assert!(
|
||||
plaintext.get::<HostRequestId>().is_none(),
|
||||
"an identifier outside the alphabet is not handed over"
|
||||
);
|
||||
|
||||
let mut peerless = Extensions::new();
|
||||
peerless.insert(None::<RemoteAddr>);
|
||||
attach_host_facts(&mut peerless);
|
||||
assert!(peerless.get::<ClientAddr>().is_none());
|
||||
}
|
||||
|
||||
/// Records what the gateway hands its authorizer about the client and the request line, and
|
||||
/// refuses.
|
||||
struct RecordingAuthorizer {
|
||||
seen: Arc<Mutex<Option<Option<ClientFacts>>>>,
|
||||
line: Arc<Mutex<Option<RequestLine>>>,
|
||||
}
|
||||
|
||||
impl Authorizer for RecordingAuthorizer {
|
||||
fn authorize_route<'a>(
|
||||
&'a self,
|
||||
context: &'a rustfs_gateway::RequestContext<'a>,
|
||||
_request: &'a AuthzRequest<'a>,
|
||||
) -> BoxFuture<'a, Decision> {
|
||||
*self.seen.lock().expect("recording lock") = Some(context.client().copied());
|
||||
*self.line.lock().expect("recording lock") = context.server_extensions().get::<RequestLine>().cloned();
|
||||
Box::pin(async { Decision::Deny })
|
||||
}
|
||||
|
||||
fn authorize_input<'a>(
|
||||
&'a self,
|
||||
_context: &'a rustfs_gateway::RequestContext<'a>,
|
||||
request: &'a InputAuthzRequest<'a>,
|
||||
) -> BoxFuture<'a, InputDecisions> {
|
||||
let decisions = request.decide_all(Decision::Deny, |_| Decision::Deny);
|
||||
Box::pin(async move { decisions })
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_authorizer_sees_the_client_facts_the_host_attached() {
|
||||
let seen = Arc::new(Mutex::new(None));
|
||||
let line = Arc::new(Mutex::new(None));
|
||||
let authorizer = RecordingAuthorizer {
|
||||
seen: Arc::clone(&seen),
|
||||
line: Arc::clone(&line),
|
||||
};
|
||||
let stack = GatewayStack::assemble_with(assembly(), Arc::new(ScriptedIam { ready: true }), root().source(), authorizer)
|
||||
.expect("assembles");
|
||||
let proxied = request(Method::GET, "/?prefix=probe")
|
||||
.extension(Some(RemoteAddr(SocketAddr::from(PEER))))
|
||||
.extension(ClientInfo::direct(SocketAddr::new(IpAddr::from(CLIENT), 443)))
|
||||
.extension(NativeTlsTransport(true));
|
||||
|
||||
let (status, _, _) = answer(&stack, proxied).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN);
|
||||
let facts = seen
|
||||
.lock()
|
||||
.expect("recording lock")
|
||||
.take()
|
||||
.expect("the authorizer was asked")
|
||||
.expect("the host attached client facts");
|
||||
assert!(facts.transport_secure, "the listener terminated TLS");
|
||||
assert_eq!(facts.client_ip, Some(IpAddr::from(CLIENT)));
|
||||
assert_eq!(facts.source_ip(), Some(IpAddr::from(CLIENT)));
|
||||
let line = line
|
||||
.lock()
|
||||
.expect("recording lock")
|
||||
.take()
|
||||
.expect("the host attached the request line");
|
||||
assert_eq!(line.method, Method::GET);
|
||||
assert_eq!(line.uri.path(), "/");
|
||||
assert_eq!(line.uri.query(), Some("prefix=probe"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_authorizer_sees_a_cleartext_transport_as_cleartext() {
|
||||
let seen = Arc::new(Mutex::new(None));
|
||||
let line = Arc::new(Mutex::new(None));
|
||||
let authorizer = RecordingAuthorizer {
|
||||
seen: Arc::clone(&seen),
|
||||
line: Arc::clone(&line),
|
||||
};
|
||||
let stack = GatewayStack::assemble_with(assembly(), Arc::new(ScriptedIam { ready: true }), root().source(), authorizer)
|
||||
.expect("assembles");
|
||||
|
||||
let (status, _, _) = answer(&stack, request(Method::GET, "/").extension(NativeTlsTransport(false))).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN);
|
||||
let facts = seen
|
||||
.lock()
|
||||
.expect("recording lock")
|
||||
.take()
|
||||
.expect("the authorizer was asked")
|
||||
.expect("the transport declaration alone makes the facts present");
|
||||
assert!(!facts.transport_secure);
|
||||
assert_eq!(facts.client_ip, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_anonymous_list_buckets_is_denied() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (status, _, body) = answer(&stack, request(Method::GET, "/")).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN);
|
||||
assert!(body.contains("<Code>AccessDenied</Code>"), "{body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unregistered_operation_is_not_implemented() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
for (method, target) in [
|
||||
(Method::GET, "/bucket/key"),
|
||||
(Method::PUT, "/bucket"),
|
||||
(Method::DELETE, "/bucket/key"),
|
||||
] {
|
||||
let (status, _, body) = answer(&stack, request(method.clone(), target)).await;
|
||||
|
||||
assert_eq!(status, StatusCode::NOT_IMPLEMENTED, "{method} {target}: {body}");
|
||||
assert!(body.contains("<Code>NotImplemented</Code>"), "{method} {target}: {body}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_wrong_signature_is_a_signature_mismatch() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), ROOT_ACCESS_KEY)).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN);
|
||||
assert!(body.contains("<Code>SignatureDoesNotMatch</Code>"), "{body}");
|
||||
assert!(!body.contains(ROOT_SECRET_KEY), "{body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unknown_access_key_is_refused_as_the_caller_s_fault() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), "AKIDUNKNOWN0001")).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "{body}");
|
||||
assert!(body.contains("<Code>InvalidAccessKeyId</Code>"), "{body}");
|
||||
}
|
||||
|
||||
/// Injected: a credential provider that returns an error for the lookup of a non-root key.
|
||||
///
|
||||
/// The pinned gateway's SigV4 authenticator answers a provider fault exactly as it answers an
|
||||
/// unknown key, `403 InvalidAccessKeyId`; the legacy stack answers a failed lookup `500`. Ledger
|
||||
/// entry R-068 (rustfs/backlog#2684): rustfs/gateway#1585 adds an opt-in switch that answers the
|
||||
/// fault as one. When the pin includes it and the assembly sets it, this test expects `500`.
|
||||
/// Until then the status is pinned, and what holds on both stacks is that nothing is served.
|
||||
#[tokio::test]
|
||||
async fn a_credential_lookup_that_fails_is_refused_and_never_served() {
|
||||
let stack = stack_over(ScriptedIam { ready: false });
|
||||
|
||||
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), "AKIDUSER0001")).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "{body}");
|
||||
assert!(body.contains("<Code>InvalidAccessKeyId</Code>"), "{body}");
|
||||
assert!(!body.contains("ListAllMyBucketsResult"), "{body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_root_key_verifies_while_iam_is_unavailable() {
|
||||
let stack = stack_over(ScriptedIam { ready: false });
|
||||
|
||||
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), ROOT_ACCESS_KEY)).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "the root secret is known without IAM: {body}");
|
||||
assert!(body.contains("<Code>SignatureDoesNotMatch</Code>"), "{body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_answer_carries_the_request_identifier_rustfs_assigned() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (_, headers, _) = answer(&stack, request(Method::GET, "/").extension(rustfs_request_context(REQUEST_ID))).await;
|
||||
|
||||
assert_eq!(headers.get("x-amz-request-id").and_then(|value| value.to_str().ok()), Some(REQUEST_ID));
|
||||
assert_eq!(headers.get("x-request-id").and_then(|value| value.to_str().ok()), Some(REQUEST_ID));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn without_a_rustfs_identifier_the_gateway_mints_its_own() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (_, headers, _) = answer(&stack, request(Method::GET, "/")).await;
|
||||
|
||||
let minted = headers
|
||||
.get("x-amz-request-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("every answer names its request");
|
||||
assert!(!minted.is_empty());
|
||||
assert_ne!(minted, REQUEST_ID);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_gateway_writes_no_cors_header() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
let browser = |builder: http::request::Builder| builder.header("origin", "https://app.example.test");
|
||||
|
||||
for (method, target) in [(Method::GET, "/"), (Method::GET, "/bucket/key"), (Method::PUT, "/bucket/key")] {
|
||||
let (_, headers, _) = answer(&stack, browser(request(method.clone(), target))).await;
|
||||
|
||||
let cors: Vec<&str> = headers
|
||||
.keys()
|
||||
.map(http::HeaderName::as_str)
|
||||
.filter(|name| name.starts_with("access-control-"))
|
||||
.collect();
|
||||
assert!(cors.is_empty(), "{method} {target}: {cors:?}");
|
||||
}
|
||||
}
|
||||
@@ -18,9 +18,6 @@ use crate::app::object::request_body::{BodyReadControl, ObservedBody};
|
||||
use crate::auth::IAMAuth;
|
||||
use crate::auth_keystone;
|
||||
use crate::config;
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
use crate::server::gateway_stack::{GatewayAssembly, GatewayStack, keystone_enabled_in_configuration};
|
||||
use crate::server::stack::{MetadataListing, S3StackService, WebsiteHosts, select_stack};
|
||||
use crate::server::{
|
||||
Http12ShutdownOutcome, ReadinessGateLayer, RemoteAddr, ShutdownHandle,
|
||||
compress::{HttpCompressionConfig, PathAwareHttpCompressionPredicate, PathCategoryInjectionLayer},
|
||||
@@ -262,7 +259,6 @@ const EVENT_HTTP_TRANSPORT_FAILED: &str = "http_transport_failed";
|
||||
const EVENT_SOCKET_FALLBACK: &str = "socket_fallback";
|
||||
const EVENT_HTTP_BIND_FAILED: &str = "http_bind_failed";
|
||||
const EVENT_HTTP_STARTUP_ENDPOINTS: &str = "http_startup_endpoints";
|
||||
const EVENT_S3_STACK_SELECTED: &str = "s3_stack_selected";
|
||||
const EVENT_HTTP_HOST_ROUTING: &str = "http_host_routing";
|
||||
const EVENT_HTTP_COMPRESSION_STATE: &str = "http_compression_state";
|
||||
const EVENT_API_RATE_LIMIT_STATE: &str = "api_rate_limit_state";
|
||||
@@ -1464,41 +1460,6 @@ fn control_request_body(body: s3s::Body) -> ControlRequestBody {
|
||||
ControlRequestBody(body)
|
||||
}
|
||||
|
||||
/// The S3 service mounted under the middleware: the legacy stack or the gateway stack, selected
|
||||
/// at startup (`server/stack.rs`).
|
||||
type S3EdgeService = S3StackService<S3Service, GatewayEdgeService>;
|
||||
|
||||
/// A build without the `gateway-stack` feature has no gateway stack to select, so this side of
|
||||
/// [`S3EdgeService`] is never constructed.
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
type GatewayEdgeService = S3Service;
|
||||
|
||||
/// The gateway stack where the legacy service is mounted: the same response and error types, so
|
||||
/// the layers around it do not change with the stack.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
#[derive(Clone)]
|
||||
struct GatewayEdgeService(GatewayStack);
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
impl<B> Service<HttpRequest<B>> for GatewayEdgeService
|
||||
where
|
||||
B: http_body::Body<Data = Bytes> + Send + 'static,
|
||||
B::Error: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
type Response = Response<s3s::Body>;
|
||||
type Error = s3s::HttpError;
|
||||
type Future = Pin<Box<dyn Future<Output = std::result::Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, request: HttpRequest<B>) -> Self::Future {
|
||||
let answer = self.0.call(request);
|
||||
Box::pin(async move { Ok(answer.await.map(s3s::Body::http_body_unsync)) })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ControlBodyDigestService<S> {
|
||||
inner: S,
|
||||
@@ -2348,10 +2309,6 @@ async fn start_http_server_inner(
|
||||
let rate_limit_vh_domains: Vec<String> = host_domain_sets.iter().flatten().cloned().collect();
|
||||
// Same domain set for resolving the bucket of virtual-hosted-style CORS requests.
|
||||
let vh_domain_hosts = rate_limit_vh_domain_hosts(rate_limit_vh_domains.clone());
|
||||
// The metadata route's own domains, also handed to the stack dispatcher (rustfs/backlog#2734
|
||||
// T2.11) so the gateway stack leaves the legacy service exactly the requests it serves there.
|
||||
let website_route_domains = website_host_domains(config)?.unwrap_or_default();
|
||||
let metadata_listing = MetadataListing::new(host_domain_sets.as_deref())?;
|
||||
|
||||
// Setup S3 service
|
||||
// This project uses the S3S library to implement S3 services
|
||||
@@ -2369,13 +2326,15 @@ async fn start_http_server_inner(
|
||||
.map(MultiDomain::new)
|
||||
.transpose()
|
||||
.map_err(Error::other)?;
|
||||
let website_host_sets = website_host_domains(config)?;
|
||||
let website_route_domains = website_host_sets.unwrap_or_default();
|
||||
|
||||
b.set_auth(IAMAuth::with_server_context(access_key, secret_key, server_ctx.clone()));
|
||||
b.set_access(store);
|
||||
b.set_route(storage::metadata_route::with_metadata_route(
|
||||
admin::make_admin_route(config.console_enable, admin_server_ctx)?,
|
||||
metadata_route_host,
|
||||
website_route_domains.clone(),
|
||||
website_route_domains,
|
||||
protocol,
|
||||
Arc::clone(&server_ctx),
|
||||
));
|
||||
@@ -2493,54 +2452,6 @@ async fn start_http_server_inner(
|
||||
rustfs_config::ENV_HTTP_REQUEST_BODY_READ_TIMEOUT,
|
||||
rustfs_config::DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT,
|
||||
));
|
||||
|
||||
// The S3 stack is selected once, for the whole server. A gateway stack that cannot be
|
||||
// assembled fails startup here; the legacy stack is never started in its place.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
let assemble_gateway = || -> Result<GatewayEdgeService> {
|
||||
let stack = GatewayStack::assemble(GatewayAssembly {
|
||||
server_ctx: Arc::clone(&server_ctx),
|
||||
root_access_key: config.access_key.clone(),
|
||||
root_secret_key: config.secret_key.clone(),
|
||||
region: config.region.clone(),
|
||||
server_domains: vh_domain_hosts.to_vec(),
|
||||
body_read_timeout: http_request_body_read_timeout,
|
||||
keystone_enabled: keystone_enabled_in_configuration(),
|
||||
})
|
||||
.map_err(Error::other)?;
|
||||
info!(
|
||||
event = EVENT_S3_STACK_SELECTED,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
stack = "gateway",
|
||||
registered_operations = stack.registered_operations().len(),
|
||||
"S3 stack selected"
|
||||
);
|
||||
Ok(GatewayEdgeService(stack))
|
||||
};
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
let assemble_gateway = || {
|
||||
Err(Error::new(
|
||||
std::io::ErrorKind::Unsupported,
|
||||
"the gateway S3 stack is not built into this binary",
|
||||
))
|
||||
};
|
||||
let s3_service: S3EdgeService = select_stack(
|
||||
config.s3_stack,
|
||||
s3_service,
|
||||
WebsiteHosts::new(website_route_domains),
|
||||
metadata_listing,
|
||||
assemble_gateway,
|
||||
)?;
|
||||
if s3_service.selected() == crate::config::S3Stack::Legacy {
|
||||
info!(
|
||||
event = EVENT_S3_STACK_SELECTED,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
stack = "legacy",
|
||||
"S3 stack selected"
|
||||
);
|
||||
}
|
||||
#[cfg(feature = "http3")]
|
||||
let http3_server = if let Some(server_config) = http3_server_config {
|
||||
let h3_s3_service = s3_service.clone();
|
||||
@@ -2978,7 +2889,7 @@ async fn start_http_server_inner(
|
||||
struct ConnectionContext {
|
||||
http_server: Arc<ConnBuilder<HttpTaskExecutor>>,
|
||||
task_owner: HttpTaskExecutor,
|
||||
s3_service: S3EdgeService,
|
||||
s3_service: S3Service,
|
||||
compression_config: HttpCompressionConfig,
|
||||
is_console: bool,
|
||||
/// Whether `RUSTFS_SERVER_DOMAINS` is configured (i.e. s3s virtual-hosted-style routing is active).
|
||||
|
||||
@@ -16,10 +16,6 @@ mod audit;
|
||||
mod compress;
|
||||
pub mod cors;
|
||||
mod event;
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
mod gateway_handlers;
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
mod gateway_stack;
|
||||
mod health;
|
||||
mod http;
|
||||
mod hybrid;
|
||||
@@ -33,7 +29,6 @@ mod runtime;
|
||||
pub(crate) mod runtime_sources;
|
||||
mod service_state;
|
||||
mod ssec_transport;
|
||||
mod stack;
|
||||
pub mod tls_material;
|
||||
pub(crate) mod tls_publish;
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ fn carries_ssec_headers(headers: &HeaderMap) -> bool {
|
||||
/// protocol is only consulted as a second source because the trusted-proxy
|
||||
/// layer has already decided whether the peer is allowed to assert it; a
|
||||
/// request that arrives direct carries no such assertion.
|
||||
pub(crate) fn is_secure_transport(connection_is_tls: bool, client_info: Option<&ClientInfo>) -> bool {
|
||||
fn is_secure_transport(connection_is_tls: bool, client_info: Option<&ClientInfo>) -> bool {
|
||||
if connection_is_tls {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -1,379 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The S3 protocol stack one server runs, selected once at startup (rustfs/backlog#2734 T2.5).
|
||||
//!
|
||||
//! Responsible for: [`S3StackService`], the service mounted where the legacy S3 service used to
|
||||
//! be, and [`select_stack`], the only place a stack is chosen. The whole stack is one or the
|
||||
//! other: no request is routed between stacks by its operation, and a request the gateway stack
|
||||
//! refuses is answered by that refusal and never retried on the legacy stack.
|
||||
//! NOT responsible for: assembling either stack (`server/http.rs` builds the legacy service,
|
||||
//! `server/gateway_stack.rs` the gateway one), or the middleware around them.
|
||||
//! Upstream: `server/http.rs`. Downstream: the legacy S3 service and the gateway stack.
|
||||
//!
|
||||
//! This file names neither stack's types: it is generic over two services with one response
|
||||
//! type, so `server/http.rs` decides what they are and how their bodies meet.
|
||||
//!
|
||||
//! Two S3-shaped families are interim legacy routes, each in its own file so that the PR which
|
||||
//! migrates one deletes only that file (rustfs/backlog#2734 T2.11): website-host requests
|
||||
//! (`stack/website.rs`) and the `?metadata=true` listings (`stack/metadata.rs`).
|
||||
|
||||
mod metadata;
|
||||
mod website;
|
||||
|
||||
use super::layer::is_sts_query_request;
|
||||
use super::prefix::{HEALTH_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, has_path_prefix, is_admin_path};
|
||||
use crate::admin::console::is_console_path;
|
||||
use crate::config::S3Stack;
|
||||
use http::{HeaderMap, Method, Request, Uri};
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll, ready};
|
||||
use tower::Service;
|
||||
|
||||
pub(crate) use metadata::MetadataListing;
|
||||
pub(crate) use website::WebsiteHosts;
|
||||
|
||||
/// The MinIO-compatible health probes share this prefix.
|
||||
const MINIO_HEALTH_PREFIX: &str = "/minio/health";
|
||||
|
||||
/// The S3 service of one server: the legacy stack, or the gateway stack with the legacy router
|
||||
/// kept for the routes that are not S3 operations.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct S3StackService<L, G> {
|
||||
legacy: L,
|
||||
gateway: Option<G>,
|
||||
website: WebsiteHosts,
|
||||
metadata: MetadataListing,
|
||||
}
|
||||
|
||||
impl<L, G> S3StackService<L, G> {
|
||||
/// The stack this service runs.
|
||||
pub(crate) fn selected(&self) -> S3Stack {
|
||||
match self.gateway {
|
||||
Some(_) => S3Stack::Gateway,
|
||||
None => S3Stack::Legacy,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the service for the selected stack.
|
||||
///
|
||||
/// `assemble_gateway` runs only when the gateway stack is selected, and its failure is the
|
||||
/// failure of the whole selection: the caller refuses to start. `website` and `metadata` are the
|
||||
/// legacy service's own website domains and metadata-listing hosts, so the gateway stack sends
|
||||
/// the legacy service exactly the requests that service would serve as those two families.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Whatever `assemble_gateway` refused with. The legacy stack is never substituted.
|
||||
pub(crate) fn select_stack<L, G, E>(
|
||||
stack: S3Stack,
|
||||
legacy: L,
|
||||
website: WebsiteHosts,
|
||||
metadata: MetadataListing,
|
||||
assemble_gateway: impl FnOnce() -> Result<G, E>,
|
||||
) -> Result<S3StackService<L, G>, E> {
|
||||
let gateway = match stack {
|
||||
S3Stack::Legacy => None,
|
||||
S3Stack::Gateway => Some(assemble_gateway()?),
|
||||
};
|
||||
Ok(S3StackService {
|
||||
legacy,
|
||||
gateway,
|
||||
website,
|
||||
metadata,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the legacy router answers this request while the gateway stack is selected: website
|
||||
/// requests and the metadata listings (interim, tested first as legacy tests them), then the
|
||||
/// admin and table-catalog APIs, the console, STS, health and profiling, none of which is an S3
|
||||
/// operation. Every other request is the gateway's, whatever it answers.
|
||||
fn is_legacy_only_route(
|
||||
website: &WebsiteHosts,
|
||||
metadata: &MetadataListing,
|
||||
method: &Method,
|
||||
uri: &Uri,
|
||||
headers: &HeaderMap,
|
||||
) -> bool {
|
||||
let path = uri.path();
|
||||
website.matches(uri, headers)
|
||||
|| metadata.matches(method, uri, headers)
|
||||
|| is_admin_path(path)
|
||||
|| is_console_path(path)
|
||||
|| has_path_prefix(path, HEALTH_PREFIX)
|
||||
|| has_path_prefix(path, MINIO_HEALTH_PREFIX)
|
||||
|| path == PROFILE_CPU_PATH
|
||||
|| path == PROFILE_MEMORY_PATH
|
||||
|| is_sts_query_request(method, uri, headers)
|
||||
}
|
||||
|
||||
impl<L, G, B, R, E> Service<Request<B>> for S3StackService<L, G>
|
||||
where
|
||||
L: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
G: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
L::Future: Send + 'static,
|
||||
G::Future: Send + 'static,
|
||||
{
|
||||
type Response = http::Response<R>;
|
||||
type Error = E;
|
||||
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
if let Some(gateway) = &mut self.gateway {
|
||||
ready!(gateway.poll_ready(cx))?;
|
||||
}
|
||||
self.legacy.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, request: Request<B>) -> Self::Future {
|
||||
match &mut self.gateway {
|
||||
// REMOVED BY T2.11: the gateway serves these routes itself once they are migrated.
|
||||
Some(gateway)
|
||||
if !is_legacy_only_route(&self.website, &self.metadata, request.method(), request.uri(), request.headers()) =>
|
||||
{
|
||||
Box::pin(gateway.call(request))
|
||||
}
|
||||
_ => Box::pin(self.legacy.call(request)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::{Response, StatusCode};
|
||||
use std::convert::Infallible;
|
||||
use std::future::{Ready, ready as ready_future};
|
||||
|
||||
/// Answers every request with its own name.
|
||||
#[derive(Clone)]
|
||||
struct Named(&'static str);
|
||||
|
||||
impl Service<Request<()>> for Named {
|
||||
type Response = Response<&'static str>;
|
||||
type Error = Infallible;
|
||||
type Future = Ready<Result<Self::Response, Self::Error>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, _request: Request<()>) -> Self::Future {
|
||||
ready_future(Ok(Response::new(self.0)))
|
||||
}
|
||||
}
|
||||
|
||||
fn request(method: Method, target: &str) -> Request<()> {
|
||||
Request::builder()
|
||||
.method(method)
|
||||
.uri(target)
|
||||
.body(())
|
||||
.expect("static request")
|
||||
}
|
||||
|
||||
async fn answered_by(service: &mut S3StackService<Named, Named>, request: Request<()>) -> &'static str {
|
||||
let response = service.call(request).await.expect("infallible");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
response.into_body()
|
||||
}
|
||||
|
||||
/// The website domain the tests configure.
|
||||
const WEBSITE_DOMAIN: &str = "site.test";
|
||||
|
||||
/// `select_stack` with one website domain and path-style metadata listings, as a server
|
||||
/// with `RUSTFS_WEBSITE_DOMAINS=site.test` and no `RUSTFS_SERVER_DOMAINS` runs.
|
||||
fn select<E>(stack: S3Stack, assemble: impl FnOnce() -> Result<Named, E>) -> Result<S3StackService<Named, Named>, E> {
|
||||
select_stack(
|
||||
stack,
|
||||
Named("legacy"),
|
||||
WebsiteHosts::new(vec![WEBSITE_DOMAIN.to_owned()]),
|
||||
MetadataListing::new(None).expect("no server domain"),
|
||||
assemble,
|
||||
)
|
||||
}
|
||||
|
||||
fn gateway_stack() -> S3StackService<Named, Named> {
|
||||
select(S3Stack::Gateway, || Ok::<_, Infallible>(Named("gateway"))).expect("assembles")
|
||||
}
|
||||
|
||||
fn on_host(method: Method, host: &str, target: &str) -> Request<()> {
|
||||
Request::builder()
|
||||
.method(method)
|
||||
.uri(target)
|
||||
.header(http::header::HOST, host)
|
||||
.body(())
|
||||
.expect("static request")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_gateway_assembly_failure_fails_the_selection_instead_of_falling_back() {
|
||||
let selection = select(S3Stack::Gateway, || Err("the assembly was refused"));
|
||||
|
||||
assert_eq!(selection.err(), Some("the assembly was refused"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_selection_never_assembles_a_gateway() {
|
||||
let service = select::<&str>(S3Stack::Legacy, || panic!("the legacy stack must not assemble the gateway"))
|
||||
.expect("legacy needs no assembly");
|
||||
|
||||
assert_eq!(service.selected(), S3Stack::Legacy);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_legacy_stack_answers_every_request_itself() {
|
||||
let mut service = select(S3Stack::Legacy, || Ok::<_, Infallible>(Named("gateway"))).expect("legacy needs no assembly");
|
||||
|
||||
for target in ["/", "/bucket/key", "/rustfs/admin/v3/info", "/health"] {
|
||||
assert_eq!(answered_by(&mut service, request(Method::GET, target)).await, "legacy", "{target}");
|
||||
}
|
||||
for (host, target) in [
|
||||
("bucket.site.test", "/"),
|
||||
("127.0.0.1:9000", "/bucket?list-type=2&metadata=true"),
|
||||
] {
|
||||
assert_eq!(
|
||||
answered_by(&mut service, on_host(Method::GET, host, target)).await,
|
||||
"legacy",
|
||||
"{host} {target}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_gateway_stack_answers_every_s3_request_whatever_the_operation() {
|
||||
let mut service = gateway_stack();
|
||||
assert_eq!(service.selected(), S3Stack::Gateway);
|
||||
|
||||
for (method, target) in [
|
||||
(Method::GET, "/"),
|
||||
(Method::GET, "/bucket/key"),
|
||||
(Method::PUT, "/bucket/key"),
|
||||
(Method::DELETE, "/bucket"),
|
||||
(Method::GET, "/bucket?versions"),
|
||||
(Method::POST, "/bucket/key?uploads"),
|
||||
// Prefixes of the legacy-only routes that are bucket names, not those routes.
|
||||
(Method::GET, "/healthy/key"),
|
||||
(Method::GET, "/rustfs/administrator"),
|
||||
(Method::GET, "/minio/healthcheck"),
|
||||
(Method::GET, "/profile/cpus"),
|
||||
] {
|
||||
assert_eq!(
|
||||
answered_by(&mut service, request(method.clone(), target)).await,
|
||||
"gateway",
|
||||
"{method} {target}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_gateway_stack_leaves_the_non_s3_routes_to_the_legacy_router() {
|
||||
let mut service = gateway_stack();
|
||||
|
||||
for target in [
|
||||
"/rustfs/admin/v3/info",
|
||||
"/minio/admin/v3/info",
|
||||
"/iceberg/v1/config",
|
||||
"/health",
|
||||
"/health/ready",
|
||||
"/minio/health/live",
|
||||
"/profile/cpu",
|
||||
"/profile/memory",
|
||||
] {
|
||||
assert_eq!(answered_by(&mut service, request(Method::GET, target)).await, "legacy", "{target}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_sts_form_post_stays_on_the_legacy_router_and_a_plain_post_does_not() {
|
||||
let mut service = gateway_stack();
|
||||
let sts = Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri("/")
|
||||
.header(http::header::CONTENT_TYPE, "application/x-www-form-urlencoded")
|
||||
.body(())
|
||||
.expect("static request");
|
||||
|
||||
assert!(
|
||||
is_sts_query_request(sts.method(), sts.uri(), sts.headers()),
|
||||
"the fixture is an STS request"
|
||||
);
|
||||
assert_eq!(answered_by(&mut service, sts).await, "legacy");
|
||||
assert_eq!(answered_by(&mut service, request(Method::POST, "/")).await, "gateway");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_gateway_stack_leaves_every_website_host_request_to_the_legacy_router() {
|
||||
let mut service = gateway_stack();
|
||||
|
||||
// Whatever the method, path or query: legacy decides website hosts before anything else,
|
||||
// so a website-host request that looks like ListBuckets, an admin call or a metadata
|
||||
// listing is still a website request.
|
||||
for (method, host, target) in [
|
||||
(Method::GET, "bucket.site.test", "/"),
|
||||
(Method::HEAD, "bucket.site.test:9000", "/index.html"),
|
||||
(Method::POST, "bucket.site.test", "/"),
|
||||
(Method::GET, "site.test", "/"),
|
||||
(Method::GET, "bucket.site.test", "/rustfs/admin/v3/info"),
|
||||
(Method::GET, "bucket.site.test", "/?list-type=2&metadata=true"),
|
||||
] {
|
||||
assert_eq!(
|
||||
answered_by(&mut service, on_host(method.clone(), host, target)).await,
|
||||
"legacy",
|
||||
"{method} {host} {target}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_gateway_stack_leaves_the_metadata_listings_to_the_legacy_router() {
|
||||
let mut service = gateway_stack();
|
||||
|
||||
for target in ["/bucket?list-type=2&metadata=true", "/bucket?versions&metadata=true"] {
|
||||
assert_eq!(
|
||||
answered_by(&mut service, on_host(Method::GET, "127.0.0.1:9000", target)).await,
|
||||
"legacy",
|
||||
"{target}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_gateway_stack_keeps_every_near_miss_of_the_interim_routes() {
|
||||
let mut service = gateway_stack();
|
||||
|
||||
for (method, host, target) in [
|
||||
// Not a website host.
|
||||
(Method::GET, "127.0.0.1:9000", "/"),
|
||||
(Method::GET, "evilsite.test", "/"),
|
||||
(Method::GET, "bucket.site.test.evil.example", "/"),
|
||||
(Method::GET, "localhost:9000", "/site.test/index.html"),
|
||||
// Not a metadata listing.
|
||||
(Method::GET, "127.0.0.1:9000", "/bucket?list-type=2"),
|
||||
(Method::GET, "127.0.0.1:9000", "/bucket?versions"),
|
||||
(Method::GET, "127.0.0.1:9000", "/bucket?list-type=2&metadata=false"),
|
||||
(Method::GET, "127.0.0.1:9000", "/bucket?list-type=2&metadata="),
|
||||
(Method::GET, "127.0.0.1:9000", "/bucket/key?list-type=2&metadata=true"),
|
||||
(Method::GET, "127.0.0.1:9000", "/bucket?metadata=true"),
|
||||
(Method::PUT, "127.0.0.1:9000", "/bucket?list-type=2&metadata=true"),
|
||||
] {
|
||||
assert_eq!(
|
||||
answered_by(&mut service, on_host(method.clone(), host, target)).await,
|
||||
"gateway",
|
||||
"{method} {host} {target}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The `?metadata=true` listing extension on the gateway stack: an INTERIM route to the legacy
|
||||
//! service (rustfs/backlog#2734 T2.11, PR "W").
|
||||
//!
|
||||
//! Responsible for: [`MetadataListing`], the predicate that says whether a request is
|
||||
//! `GET /{bucket}?list-type=2&metadata=true` or `GET /{bucket}?versions&metadata=true`, the
|
||||
//! MinIO-compatible listings that carry `<UserMetadata>`. No gateway DTO can carry that element,
|
||||
//! and a gateway ListObjectsV2 or ListObjectVersions would answer the request as a plain listing
|
||||
//! and silently drop it, so while the gateway stack is selected these requests still go to the
|
||||
//! legacy service.
|
||||
//! NOT responsible for: serving or authorizing the listing (`app/metadata_route.rs` does, on
|
||||
//! legacy), or the rule itself: [`MetadataListing`] is the legacy route's own
|
||||
//! [`MetadataListingHosts`], so the two stacks cannot disagree on which requests are metadata
|
||||
//! listings, virtual-hosted ones included.
|
||||
//! Upstream: `server/stack.rs`. Downstream: `app/metadata_route.rs`.
|
||||
//!
|
||||
//! Interim. The final plan (PR "M" of the T2.11 design) serves these two listings as gateway
|
||||
//! dialect operations, authenticated by the gateway (rustfs/gateway#1588); that PR deletes this
|
||||
//! file.
|
||||
|
||||
pub(crate) use crate::app::metadata_route::MetadataListingHosts as MetadataListing;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::header::HOST;
|
||||
use http::{HeaderMap, Method};
|
||||
|
||||
fn path_style() -> MetadataListing {
|
||||
MetadataListing::new(None).expect("no server domain")
|
||||
}
|
||||
|
||||
fn virtual_hosted() -> MetadataListing {
|
||||
MetadataListing::new(Some(&["s3.test".to_owned()])).expect("valid server domain")
|
||||
}
|
||||
|
||||
fn matched(listing: &MetadataListing, method: Method, host: &str, target: &str) -> bool {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(HOST, host.parse().expect("test host header"));
|
||||
listing.matches(&method, &target.parse().expect("test target"), &headers)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn both_metadata_listings_of_a_path_style_bucket_are_matched() {
|
||||
let listing = path_style();
|
||||
|
||||
for target in [
|
||||
"/bucket?list-type=2&metadata=true",
|
||||
"/bucket?versions&metadata=true",
|
||||
"/bucket?metadata=true&list-type=2",
|
||||
"/bucket?versions=&metadata=true&prefix=a%2F",
|
||||
"/bucket/?list-type=2&metadata=true",
|
||||
// The value is form-decoded, as legacy decodes it.
|
||||
"/bucket?list-type=2&metadata=%74rue",
|
||||
] {
|
||||
assert!(matched(&listing, Method::GET, "127.0.0.1:9000", target), "{target}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_metadata_listing_of_a_virtual_hosted_bucket_root_is_matched() {
|
||||
let listing = virtual_hosted();
|
||||
|
||||
assert!(matched(&listing, Method::GET, "bucket.s3.test:9000", "/?list-type=2&metadata=true"));
|
||||
assert!(matched(&listing, Method::GET, "bucket.s3.test", "/?versions&metadata=true"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_listing_without_metadata_true_stays_on_the_gateway() {
|
||||
let listing = path_style();
|
||||
|
||||
for target in [
|
||||
"/bucket?list-type=2",
|
||||
"/bucket?versions",
|
||||
"/bucket?list-type=2&metadata=false",
|
||||
"/bucket?list-type=2&metadata=",
|
||||
"/bucket?list-type=2&metadata",
|
||||
"/bucket?list-type=2&metadata=TRUE",
|
||||
"/bucket?list-type=2&metadata=1",
|
||||
// An encoded `=` makes one parameter named `metadata=true`, with no value.
|
||||
"/bucket?list-type=2&metadata%3Dtrue",
|
||||
] {
|
||||
assert!(!matched(&listing, Method::GET, "127.0.0.1:9000", target), "{target}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_true_on_something_other_than_a_listing_stays_on_the_gateway() {
|
||||
let listing = path_style();
|
||||
|
||||
for target in [
|
||||
"/bucket?metadata=true",
|
||||
"/bucket?list-type=1&metadata=true",
|
||||
"/bucket?uploads&metadata=true",
|
||||
"/bucket?location&metadata=true",
|
||||
"/bucket",
|
||||
] {
|
||||
assert!(!matched(&listing, Method::GET, "127.0.0.1:9000", target), "{target}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_true_on_an_object_or_on_no_bucket_stays_on_the_gateway() {
|
||||
let listing = path_style();
|
||||
|
||||
for target in [
|
||||
"/bucket/key?list-type=2&metadata=true",
|
||||
"/bucket/dir/key?versions&metadata=true",
|
||||
"/?list-type=2&metadata=true",
|
||||
"/?versions&metadata=true",
|
||||
] {
|
||||
assert!(!matched(&listing, Method::GET, "127.0.0.1:9000", target), "{target}");
|
||||
}
|
||||
let listing = virtual_hosted();
|
||||
assert!(!matched(&listing, Method::GET, "bucket.s3.test", "/key?list-type=2&metadata=true"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_metadata_listing_with_another_method_stays_on_the_gateway() {
|
||||
let listing = path_style();
|
||||
|
||||
for method in [Method::HEAD, Method::PUT, Method::POST, Method::DELETE] {
|
||||
assert!(
|
||||
!matched(&listing, method.clone(), "127.0.0.1:9000", "/bucket?list-type=2&metadata=true"),
|
||||
"{method}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Static-website hosting on the gateway stack: an INTERIM route to the legacy service
|
||||
//! (rustfs/backlog#2734 T2.11, PR "W"; release blocker R-064).
|
||||
//!
|
||||
//! Responsible for: [`WebsiteHosts`], the predicate the stack dispatcher asks first, which says
|
||||
//! whether a request's `Host` names a `RUSTFS_WEBSITE_DOMAINS` domain. Such a request is a
|
||||
//! website request on legacy (index and error documents, redirects, always anonymous), and the
|
||||
//! gateway has no website implementation, so while the gateway stack is selected it still goes to
|
||||
//! the legacy service, which serves it with the released code.
|
||||
//! NOT responsible for: serving the request (`app/metadata_route.rs` does, on legacy), or the
|
||||
//! rule itself: [`WebsiteHosts::matches`] calls the legacy route's own
|
||||
//! [`is_website_request`], so the two stacks cannot disagree on which requests are website ones.
|
||||
//! Upstream: `server/stack.rs`. Downstream: `app/metadata_route.rs`.
|
||||
//!
|
||||
//! Interim. The final plan (PR "W2" of the T2.11 design) serves website requests from an
|
||||
//! s3s-free RustFS service in this same tower position, built on the owned GetObject and
|
||||
//! HeadObject use cases once that slice lands; that PR deletes this file.
|
||||
|
||||
use crate::app::metadata_route::is_website_request;
|
||||
use http::{HeaderMap, Uri};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The website domains, normalised as legacy normalises them (`RUSTFS_WEBSITE_DOMAINS`,
|
||||
/// lower-cased, no port, no trailing dot). Empty when website hosting is off.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct WebsiteHosts(Arc<[String]>);
|
||||
|
||||
impl WebsiteHosts {
|
||||
/// The same list the legacy metadata route is given.
|
||||
pub(crate) fn new(domains: Vec<String>) -> Self {
|
||||
Self(domains.into())
|
||||
}
|
||||
|
||||
/// Whether legacy serves this request as a website request.
|
||||
pub(super) fn matches(&self, uri: &Uri, headers: &HeaderMap) -> bool {
|
||||
is_website_request(uri, headers, &self.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::header::HOST;
|
||||
|
||||
fn hosts(domains: &[&str]) -> WebsiteHosts {
|
||||
WebsiteHosts::new(domains.iter().map(|domain| (*domain).to_owned()).collect())
|
||||
}
|
||||
|
||||
fn with_host(host: &[u8]) -> HeaderMap {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(HOST, http::HeaderValue::from_bytes(host).expect("test host header"));
|
||||
headers
|
||||
}
|
||||
|
||||
fn matched(website: &WebsiteHosts, host: &str, target: &str) -> bool {
|
||||
website.matches(&target.parse().expect("test target"), &with_host(host.as_bytes()))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_website_domain_or_a_bucket_under_it_is_a_website_request() {
|
||||
let website = hosts(&["site.test"]);
|
||||
|
||||
for (host, target) in [
|
||||
// The bare domain: legacy answers it 404 as a website, never as ListBuckets.
|
||||
("site.test", "/"),
|
||||
("bucket.site.test", "/"),
|
||||
("bucket.site.test", "/docs/index.html"),
|
||||
("my.dotted.bucket.site.test", "/"),
|
||||
("bucket.site.test:9000", "/"),
|
||||
("Bucket.SITE.Test", "/"),
|
||||
("bucket.site.test.", "/"),
|
||||
("bucket.site.test.:9000", "/key"),
|
||||
] {
|
||||
assert!(matched(&website, host, target), "{host} {target} must be a website request");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_configured_website_domain_is_matched() {
|
||||
let website = hosts(&["site.test", "pages.example"]);
|
||||
|
||||
assert!(matched(&website, "a.site.test", "/"));
|
||||
assert!(matched(&website, "b.pages.example", "/"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_host_is_a_website_request_when_website_hosting_is_off() {
|
||||
let website = WebsiteHosts::default();
|
||||
|
||||
for host in ["site.test", "bucket.site.test", "localhost:9000"] {
|
||||
assert!(!matched(&website, host, "/"), "{host} with no website domain configured");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_server_domain_that_is_not_a_website_domain_stays_on_the_gateway() {
|
||||
// RUSTFS_SERVER_DOMAINS=s3.test is validated never to overlap a website domain.
|
||||
let website = hosts(&["site.test"]);
|
||||
|
||||
assert!(!matched(&website, "s3.test", "/"));
|
||||
assert!(!matched(&website, "bucket.s3.test", "/key"));
|
||||
assert!(!matched(&website, "bucket.s3.test:9000", "/?list-type=2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_host_that_only_resembles_a_website_domain_stays_on_the_gateway() {
|
||||
let website = hosts(&["site.test"]);
|
||||
|
||||
for host in [
|
||||
"127.0.0.1:9000",
|
||||
"localhost:9000",
|
||||
"unrelated.example",
|
||||
// A suffix without the label boundary.
|
||||
"evilsite.test",
|
||||
"bucket.evilsite.test",
|
||||
// The domain as a prefix of another one.
|
||||
"site.test.evil.example",
|
||||
"bucket.site.test.evil.example",
|
||||
// A label of the domain, not the domain.
|
||||
"site",
|
||||
"test",
|
||||
"[::1]:9000",
|
||||
] {
|
||||
assert!(!matched(&website, host, "/"), "{host} must not be a website request");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_or_unreadable_host_stays_on_the_gateway() {
|
||||
let website = hosts(&["site.test"]);
|
||||
let target: Uri = "/".parse().expect("test target");
|
||||
|
||||
assert!(!website.matches(&target, &HeaderMap::new()), "no Host header");
|
||||
assert!(!website.matches(&target, &with_host(b"")), "an empty Host header");
|
||||
assert!(!website.matches(&target, &with_host(b"bucket .site.test")), "a Host that is no authority");
|
||||
assert!(!website.matches(&target, &with_host(b"bucket.site.\xfftest")), "a Host that is not UTF-8");
|
||||
// An absolute-form target names a website host, but legacy matches on the Host header only.
|
||||
let absolute: Uri = "http://bucket.site.test/".parse().expect("absolute target");
|
||||
assert!(!website.matches(&absolute, &with_host(b"127.0.0.1:9000")), "absolute-form target");
|
||||
}
|
||||
}
|
||||
@@ -41,8 +41,7 @@ const EVENT_SERVER_STARTING: &str = "server_starting";
|
||||
const EVENT_ACTION_CREDENTIALS_INITIALIZED: &str = "action_credentials_initialized";
|
||||
const EVENT_ACTION_CREDENTIALS_INITIALIZATION_FAILED: &str = "action_credentials_initialization_failed";
|
||||
const DEFAULT_CREDENTIALS_WARNING_MESSAGE: &str = "Detected default root credentials; set RUSTFS_ACCESS_KEY and RUSTFS_SECRET_KEY to non-default values for production deployments";
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
const GATEWAY_STACK_NOT_BUILT: &str = "RUSTFS_S3_STACK=gateway needs a binary built with the `gateway-stack` feature, which no release build enables (rustfs/backlog#2734)";
|
||||
const GATEWAY_STACK_NOT_ASSEMBLED: &str = "gateway stack is not assembled yet (rustfs/backlog#2734 T2.5)";
|
||||
|
||||
pub(crate) struct StartupListenContext {
|
||||
pub(crate) readiness: Arc<GlobalReadiness>,
|
||||
@@ -83,7 +82,12 @@ pub(crate) async fn init_startup_listen_context(
|
||||
config: &Config,
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
) -> Result<StartupListenContext> {
|
||||
refuse_unbuilt_stack(config.s3_stack)?;
|
||||
match config.s3_stack {
|
||||
S3Stack::Legacy => {}
|
||||
// REPLACED BY T2.5
|
||||
// Refuse before any listener or storage exists; never fall back to legacy.
|
||||
S3Stack::Gateway => return Err(Error::new(ErrorKind::Unsupported, GATEWAY_STACK_NOT_ASSEMBLED)),
|
||||
}
|
||||
crate::server::init_console_prefix()?;
|
||||
log_sanitized_server_config(config);
|
||||
let readiness = Arc::new(GlobalReadiness::new());
|
||||
@@ -134,18 +138,6 @@ pub(crate) async fn init_startup_listen_context(
|
||||
})
|
||||
}
|
||||
|
||||
/// Refuses a selected S3 stack this binary cannot serve, before any listener or storage exists.
|
||||
/// The selection is never replaced by another stack.
|
||||
fn refuse_unbuilt_stack(stack: S3Stack) -> Result<()> {
|
||||
match stack {
|
||||
S3Stack::Legacy => Ok(()),
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
S3Stack::Gateway => Ok(()),
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
S3Stack::Gateway => Err(Error::new(ErrorKind::Unsupported, GATEWAY_STACK_NOT_BUILT)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_embedded_startup_config(
|
||||
address: String,
|
||||
access_key: String,
|
||||
@@ -174,10 +166,6 @@ pub(crate) async fn prepare_embedded_startup_config(
|
||||
config.secret_key = secret_key.clone();
|
||||
config.region = Some(region.clone());
|
||||
config.console_enable = false;
|
||||
// The stack is selected for the whole process, so an embedded server reads the same variable
|
||||
// the binary does.
|
||||
config.s3_stack = S3Stack::from_env()?;
|
||||
refuse_unbuilt_stack(config.s3_stack)?;
|
||||
|
||||
Ok(EmbeddedStartupConfig {
|
||||
config,
|
||||
@@ -530,26 +518,6 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
#[test]
|
||||
fn a_binary_without_the_gateway_stack_refuses_to_select_it() {
|
||||
let err = super::refuse_unbuilt_stack(crate::config::S3Stack::Gateway).expect_err("the gateway stack is not built");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::Unsupported);
|
||||
assert!(err.to_string().contains("gateway-stack"), "{err}");
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
#[test]
|
||||
fn a_binary_with_the_gateway_stack_accepts_its_selection() {
|
||||
super::refuse_unbuilt_stack(crate::config::S3Stack::Gateway).expect("the gateway stack is built");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_stack_is_always_selectable() {
|
||||
super::refuse_unbuilt_stack(crate::config::S3Stack::Legacy).expect("legacy is the default stack");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn init_startup_listen_context_rejects_console_s3_port_conflict() {
|
||||
let mut config = Config::new("127.0.0.1:9001", vec!["/tmp/rustfs-data".to_string()]);
|
||||
|
||||
+541
-398
File diff suppressed because it is too large
Load Diff
@@ -20,10 +20,8 @@
|
||||
//! counted and the count is carried on the next entry that is let through.
|
||||
|
||||
use crate::module_switches::is_audit_module_enabled;
|
||||
use crate::storage::helper::{
|
||||
AuditRequestView, DenialRequest, DenialScope, authorization_denial_entry, authorization_denial_entry_for_view,
|
||||
spawn_background_with_context,
|
||||
};
|
||||
use crate::storage::access::{ReqInfo, request_context_from_req};
|
||||
use crate::storage::helper::{DenialRequest, DenialScope, authorization_denial_entry, spawn_background_with_context};
|
||||
use crate::storage::request_context::RequestContext;
|
||||
use rustfs_audit::{AuditEntry, global::AuditLogger};
|
||||
use rustfs_policy::policy::action::Action;
|
||||
@@ -186,32 +184,21 @@ impl DenialAuditor {
|
||||
scope: DenialScope,
|
||||
now: Instant,
|
||||
) {
|
||||
self.audit_view_at(&AuditRequestView::legacy(req), now, || {
|
||||
authorization_denial_entry(req, action, reason, scope)
|
||||
});
|
||||
}
|
||||
|
||||
/// Audits a denial for a request an owned edge decoded into `view`.
|
||||
pub(crate) fn audit_owned(&self, view: &AuditRequestView<'_>, action: &Action, reason: &'static str, scope: DenialScope) {
|
||||
self.audit_view_at(view, (self.clock)(), || authorization_denial_entry_for_view(view, action, reason, scope));
|
||||
}
|
||||
|
||||
fn audit_view_at(&self, req: &AuditRequestView<'_>, now: Instant, build_entry: impl FnOnce() -> AuditEntry) {
|
||||
if !(self.enabled)() {
|
||||
return;
|
||||
}
|
||||
let anonymous = req.req_info.is_none_or(|info| info.cred.is_none());
|
||||
let anonymous = req.extensions.get::<ReqInfo>().is_none_or(|info| info.cred.is_none());
|
||||
let Admission::Emit { suppressed } = self.limiter.admit(anonymous, now) else {
|
||||
return;
|
||||
};
|
||||
let mut entry = build_entry();
|
||||
let mut entry = authorization_denial_entry(req, action, reason, scope);
|
||||
if suppressed > 0 {
|
||||
entry
|
||||
.tags
|
||||
.get_or_insert_with(Default::default)
|
||||
.insert(SUPPRESSED_DENIALS_TAG.to_string(), Value::from(suppressed));
|
||||
}
|
||||
(self.emit)(req.request_context(), entry);
|
||||
(self.emit)(request_context_from_req(req), entry);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -45,7 +45,6 @@ use crate::storage::validate_website_configuration;
|
||||
use crate::table_catalog;
|
||||
use http::StatusCode;
|
||||
use metrics::{counter, histogram};
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_buckets as gateway_list_buckets;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_madmin::{SITE_REPL_API_VERSION, SRBucketMeta};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
@@ -741,15 +740,6 @@ impl FS {
|
||||
}
|
||||
}
|
||||
|
||||
/// A ListBuckets value the s3s edge and the gateway's types disagree on. The seam converts every
|
||||
/// member both sides hold, so this is a defect in the conversion, never the caller's fault.
|
||||
fn list_buckets_seam_error(error: rustfs_gateway_types::compat::ConversionError) -> S3Error {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("failed to convert ListBuckets through the gateway compatibility seam: {error}"),
|
||||
)
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl S3 for FS {
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
@@ -1606,17 +1596,7 @@ impl S3 for FS {
|
||||
// List buckets not associated with a bucket, give it bucket label "*" to denote "all".
|
||||
record_s3_op(S3Operation::ListBuckets);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
// The use case speaks the envelope and the gateway's types; this edge only converts.
|
||||
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
|
||||
let input = gateway_list_buckets::input_from_s3s(req.input).map_err(list_buckets_seam_error)?;
|
||||
let output = usecase.execute_list_buckets(envelope, input).await?;
|
||||
let mut output = gateway_list_buckets::output_to_s3s(output).map_err(list_buckets_seam_error)?;
|
||||
// The seam writes a listing without buckets as an absent member. This edge has always
|
||||
// answered the element, empty or not, and clients index the answer by it.
|
||||
if output.buckets.is_none() {
|
||||
output.buckets = Some(Vec::new());
|
||||
}
|
||||
Ok(S3Response::new(output))
|
||||
usecase.execute_list_buckets(req).await
|
||||
}
|
||||
|
||||
async fn list_multipart_uploads(
|
||||
|
||||
@@ -19,7 +19,7 @@ use crate::storage::request_context::RequestContext;
|
||||
use crate::storage::sse::KmsRequestAuditScope;
|
||||
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
|
||||
use hashbrown::HashMap;
|
||||
use http::{HeaderMap, StatusCode, Uri};
|
||||
use http::StatusCode;
|
||||
use metrics::counter;
|
||||
use rustfs_audit::{
|
||||
ApiDetails, AuditEntry, ObjectVersion,
|
||||
@@ -38,64 +38,12 @@ use rustfs_utils::http::headers::AMZ_REQUEST_ID;
|
||||
use s3s::{S3Request, S3Response, S3Result};
|
||||
use serde_json::Value;
|
||||
use std::future::Future;
|
||||
use std::net::SocketAddr;
|
||||
use tokio::runtime::{Builder, Handle};
|
||||
|
||||
/// Request type seen by the denial auditor, so `denial_audit` stays off the direct s3s surface.
|
||||
pub(crate) type DenialRequest<T> = S3Request<T>;
|
||||
use tracing::{Instrument, info_span, warn};
|
||||
|
||||
/// The request fields used to build an audit entry, shared by both request edges.
|
||||
pub(crate) struct AuditRequestView<'a> {
|
||||
headers: &'a HeaderMap,
|
||||
uri: &'a Uri,
|
||||
pub(crate) req_info: Option<&'a ReqInfo>,
|
||||
client_info: Option<&'a ClientInfo>,
|
||||
remote_addr: Option<SocketAddr>,
|
||||
request_context: Option<&'a RequestContext>,
|
||||
}
|
||||
|
||||
impl<'a> AuditRequestView<'a> {
|
||||
pub(crate) fn legacy<T>(req: &'a S3Request<T>) -> Self {
|
||||
Self {
|
||||
headers: &req.headers,
|
||||
uri: &req.uri,
|
||||
req_info: req.extensions.get::<ReqInfo>(),
|
||||
client_info: req.extensions.get::<ClientInfo>(),
|
||||
remote_addr: req
|
||||
.extensions
|
||||
.get::<Option<RemoteAddr>>()
|
||||
.and_then(|addr| addr.map(|addr| addr.0)),
|
||||
request_context: req.extensions.get::<RequestContext>(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The view of a request whose facts an owned edge already decoded.
|
||||
pub(crate) fn from_facts(
|
||||
headers: &'a HeaderMap,
|
||||
uri: &'a Uri,
|
||||
req_info: Option<&'a ReqInfo>,
|
||||
client_info: Option<&'a ClientInfo>,
|
||||
remote_addr: Option<SocketAddr>,
|
||||
request_context: Option<&'a RequestContext>,
|
||||
) -> Self {
|
||||
Self {
|
||||
headers,
|
||||
uri,
|
||||
req_info,
|
||||
client_info,
|
||||
remote_addr,
|
||||
request_context,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn request_context(&self) -> Option<RequestContext> {
|
||||
self.request_context
|
||||
.cloned()
|
||||
.or_else(|| self.req_info.and_then(|info| info.request_context.clone()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
use super::storage_api::ecstore_diagnostic;
|
||||
use crate::storage::storage_api::helper_consumer::StorageObjectInfo as ObjectInfo;
|
||||
@@ -179,27 +127,32 @@ pub(crate) fn build_event_resp_elements<T>(response: &S3Response<T>, request_id:
|
||||
|
||||
/// Builds the audit entry shared by every request-scoped audit source: caller identity,
|
||||
/// client address and the request line, without any outcome.
|
||||
fn request_audit_builder(
|
||||
req: &AuditRequestView<'_>,
|
||||
fn request_audit_builder<T>(
|
||||
req: &S3Request<T>,
|
||||
req_info: Option<&ReqInfo>,
|
||||
event: EventName,
|
||||
trigger: &str,
|
||||
api: ApiDetails,
|
||||
request_id: &str,
|
||||
) -> AuditEntryBuilder {
|
||||
let remote_host = req
|
||||
.client_info
|
||||
.extensions
|
||||
.get::<ClientInfo>()
|
||||
.map(|info| info.real_ip.to_string())
|
||||
.or_else(|| req.remote_addr.map(|addr| addr.ip().to_string()))
|
||||
.or_else(|| {
|
||||
req.extensions
|
||||
.get::<Option<RemoteAddr>>()
|
||||
.and_then(|addr| addr.map(|addr| addr.0.ip().to_string()))
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let mut builder = AuditEntryBuilder::new("1.0", event, trigger, api)
|
||||
.remote_host(remote_host)
|
||||
.user_agent(get_request_user_agent(req.headers))
|
||||
.req_host(get_request_host(req.headers))
|
||||
.user_agent(get_request_user_agent(&req.headers))
|
||||
.req_host(get_request_host(&req.headers))
|
||||
.req_path(req.uri.path().to_string())
|
||||
.req_query(extract_params_header(req.headers))
|
||||
.req_query(extract_params_header(&req.headers))
|
||||
.request_id(request_id);
|
||||
if let Some(cred) = req
|
||||
.req_info
|
||||
if let Some(cred) = req_info
|
||||
.and_then(|info| info.cred.as_ref())
|
||||
.filter(|cred| !cred.access_key.is_empty())
|
||||
{
|
||||
@@ -421,9 +374,8 @@ impl OperationHelper {
|
||||
let request_context = request_context.unwrap_or_else(|| RequestContext::from_external_headers(&req.headers));
|
||||
let request_id = request_context.request_id.clone();
|
||||
|
||||
let audit_builder = audit_enabled.then(|| {
|
||||
request_audit_builder(&AuditRequestView::legacy(req), event, trigger, api_builder.clone().build(), &request_id)
|
||||
});
|
||||
let audit_builder =
|
||||
audit_enabled.then(|| request_audit_builder(req, req_info, event, trigger, api_builder.clone().build(), &request_id));
|
||||
|
||||
let event_object = ObjectInfo {
|
||||
bucket: bucket.clone(),
|
||||
@@ -650,16 +602,7 @@ pub(crate) fn authorization_denial_entry<T>(
|
||||
reason: &'static str,
|
||||
scope: DenialScope,
|
||||
) -> AuditEntry {
|
||||
authorization_denial_entry_for_view(&AuditRequestView::legacy(req), action, reason, scope)
|
||||
}
|
||||
|
||||
pub(crate) fn authorization_denial_entry_for_view(
|
||||
req: &AuditRequestView<'_>,
|
||||
action: &Action,
|
||||
reason: &'static str,
|
||||
scope: DenialScope,
|
||||
) -> AuditEntry {
|
||||
let req_info = req.req_info;
|
||||
let req_info = req.extensions.get::<ReqInfo>();
|
||||
let path = req.uri.path().trim_start_matches('/');
|
||||
let mut segs = path.splitn(2, '/');
|
||||
let path_bucket = segs.next().unwrap_or("");
|
||||
@@ -688,18 +631,23 @@ pub(crate) fn authorization_denial_entry_for_view(
|
||||
api = api.object(object);
|
||||
}
|
||||
|
||||
let request_context = req
|
||||
.request_context()
|
||||
.unwrap_or_else(|| RequestContext::from_external_headers(req.headers));
|
||||
let request_context = request_context_from_req(req).unwrap_or_else(|| RequestContext::from_external_headers(&req.headers));
|
||||
let mut tags = HashMap::new();
|
||||
tags.insert("denialReason".to_string(), Value::String(reason.to_string()));
|
||||
if let DenialScope::Keys(denied) = scope {
|
||||
tags.insert("deniedKeys".to_string(), Value::from(denied));
|
||||
}
|
||||
request_audit_builder(req, EventName::AuditS3Operation, action_name, api.build(), &request_context.request_id)
|
||||
.error("Access Denied")
|
||||
.tags(tags)
|
||||
.build()
|
||||
request_audit_builder(
|
||||
req,
|
||||
req_info,
|
||||
EventName::AuditS3Operation,
|
||||
action_name,
|
||||
api.build(),
|
||||
&request_context.request_id,
|
||||
)
|
||||
.error("Access Denied")
|
||||
.tags(tags)
|
||||
.build()
|
||||
}
|
||||
|
||||
/// Runs `handler` on `req` and records the outcome as an audit-only entry for `op`.
|
||||
@@ -1487,7 +1435,8 @@ mod tests {
|
||||
..Default::default()
|
||||
});
|
||||
let entry = super::request_audit_builder(
|
||||
&super::AuditRequestView::legacy(&req),
|
||||
&req,
|
||||
req.extensions.get::<ReqInfo>(),
|
||||
EventName::ObjectAccessedGet,
|
||||
"GetObject",
|
||||
rustfs_audit::ApiDetails::default(),
|
||||
|
||||
@@ -13,14 +13,15 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::s3_api::common::rustfs_owner;
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::contract::list::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::contract::{
|
||||
bucket::BucketInfo,
|
||||
list::{ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info},
|
||||
};
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::to_s3s_xml_etag;
|
||||
use percent_encoding::percent_decode_str;
|
||||
use s3s::dto::{
|
||||
CommonPrefix, DeleteMarkerEntry, EncodingType, ListObjectVersionsOutput, ListObjectsOutput, ListObjectsV2Output, Object,
|
||||
ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Timestamp,
|
||||
Bucket, CommonPrefix, DeleteMarkerEntry, EncodingType, ListBucketsOutput, ListObjectVersionsOutput, ListObjectsOutput,
|
||||
ListObjectsV2Output, Object, ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Timestamp,
|
||||
};
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
use tracing::debug;
|
||||
@@ -125,6 +126,23 @@ pub(crate) fn build_list_objects_output(v2: ListObjectsV2Output, request_marker:
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_list_buckets_output(bucket_infos: &[BucketInfo]) -> ListBucketsOutput {
|
||||
let buckets: Vec<Bucket> = bucket_infos
|
||||
.iter()
|
||||
.map(|bucket_info| Bucket {
|
||||
creation_date: Some(Timestamp::from(bucket_info.created.unwrap_or(time::OffsetDateTime::UNIX_EPOCH))),
|
||||
name: Some(bucket_info.name.clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
|
||||
ListBucketsOutput {
|
||||
buckets: Some(buckets),
|
||||
owner: Some(rustfs_owner()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_list_object_versions_params(
|
||||
prefix: Option<String>,
|
||||
delimiter: Option<String>,
|
||||
@@ -432,16 +450,19 @@ fn calculate_next_marker(v2: &ListObjectsV2Output) -> Option<String> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
ListObjectVersionsInfo, ListObjectVersionsParams, ListObjectsV2Info, build_list_object_versions_output,
|
||||
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
|
||||
parse_list_object_versions_params, parse_list_objects_v2_params, to_s3s_xml_etag,
|
||||
ListObjectVersionsInfo, ListObjectVersionsParams, ListObjectsV2Info, build_list_buckets_output,
|
||||
build_list_object_versions_output, build_list_objects_output, build_list_objects_v2_output,
|
||||
encode_list_objects_v2_continuation_token, parse_list_object_versions_params, parse_list_objects_v2_params,
|
||||
to_s3s_xml_etag,
|
||||
};
|
||||
use crate::storage::s3_api::common::rustfs_owner;
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::StorageObjectInfo as ObjectInfo;
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::contract::bucket::BucketInfo;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::{CommonPrefix, EncodingType, ListObjectsV2Output, Object};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
@@ -504,6 +525,36 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_list_buckets_output_maps_bucket_infos_and_owner() {
|
||||
let bucket_infos = vec![
|
||||
BucketInfo {
|
||||
name: "bucket-a".to_string(),
|
||||
created: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
},
|
||||
BucketInfo {
|
||||
name: "bucket-b".to_string(),
|
||||
created: None,
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
|
||||
let output = build_list_buckets_output(&bucket_infos);
|
||||
let buckets = output.buckets.as_ref().expect("buckets should be present");
|
||||
let owner = output.owner.as_ref().expect("owner should be present");
|
||||
|
||||
assert_eq!(buckets.len(), 2);
|
||||
assert_eq!(buckets[0].name.as_deref(), Some("bucket-a"));
|
||||
assert_eq!(buckets[0].creation_date, Some(s3s::dto::Timestamp::from(OffsetDateTime::UNIX_EPOCH)));
|
||||
assert_eq!(buckets[1].name.as_deref(), Some("bucket-b"));
|
||||
assert_eq!(buckets[1].creation_date, Some(s3s::dto::Timestamp::from(OffsetDateTime::UNIX_EPOCH)));
|
||||
|
||||
let expected_owner = rustfs_owner();
|
||||
assert_eq!(owner.display_name, expected_owner.display_name);
|
||||
assert_eq!(owner.id, expected_owner.id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_objects_marker_echoes_request_value() {
|
||||
let output = build_list_objects_output(ListObjectsV2Output::default(), Some("m-1".to_string()));
|
||||
|
||||
@@ -25,14 +25,11 @@ impl RequestEnvelope {
|
||||
/// Copies every request fact of `req` the app layer reads.
|
||||
pub fn from_s3s<T>(req: &S3Request<T>) -> Self {
|
||||
let credentials = req.credentials.as_ref().map(|credentials| {
|
||||
let session_token = get_session_token(&req.uri, &req.headers).map(str::to_owned);
|
||||
// An empty secret is not a key: carry it as absent so every reader fails closed.
|
||||
RequestCredentials::new(
|
||||
credentials.access_key.clone(),
|
||||
credentials.secret_key.expose().to_owned(),
|
||||
session_token.clone(),
|
||||
get_session_token(&req.uri, &req.headers).map(str::to_owned),
|
||||
)
|
||||
.unwrap_or_else(|_| RequestCredentials::without_secret_key(credentials.access_key.clone(), session_token))
|
||||
});
|
||||
let head = RequestHead {
|
||||
method: req.method.clone(),
|
||||
@@ -181,7 +178,7 @@ mod tests {
|
||||
assert_eq!(envelope.headers(), &req.headers);
|
||||
let credentials = envelope.credentials().expect("credentials");
|
||||
assert_eq!(credentials.access_key(), "AKIAENVELOPE");
|
||||
assert_eq!(credentials.secret_key(), Some("envelope-secret"));
|
||||
assert_eq!(credentials.secret_key(), "envelope-secret");
|
||||
assert_eq!(credentials.session_token(), Some("header-token"));
|
||||
assert_eq!(envelope.region(), Some("eu-west-3"));
|
||||
assert_eq!(envelope.service(), Some("s3"));
|
||||
|
||||
@@ -147,36 +147,17 @@ pub(crate) use super::sse::{
|
||||
|
||||
pub(crate) mod access_consumer {
|
||||
pub(crate) use super::super::access::{
|
||||
AuthorizationRequest, BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard,
|
||||
ListBucketsAuthzRequest, MultipartStagingOperation, ObjectTagConditions, OdmReadGenerationGuard,
|
||||
PendingDeleteBucketGenerationGuard, PostObjectPolicyHeaders, PostObjectRequestMarker, ReqInfo, RequestedCopySource,
|
||||
RequestedObjectLegalHold, RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags,
|
||||
StagedMultipartPartAuthorization, TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess,
|
||||
TableDataPlaneListCursorPosition, TableDataPlanePublicationGuards, apply_bucket_generation_guard,
|
||||
apply_copy_source_bucket_generation_guard, authorize_copy_object_tags, authorize_request, authorize_request_core,
|
||||
bucket_config_mutation_incarnation, delete_object_authorize_action, has_bypass_governance_header,
|
||||
load_bucket_generation_from_store, log_list_buckets_iam_implicit_deny, odm_read_generation,
|
||||
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller,
|
||||
replication_proxy_allowed, replication_request_authorized, req_info_mut, req_info_ref,
|
||||
retained_table_data_plane_publication_guards, set_requested_object_lock_retain_until,
|
||||
BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard, MultipartStagingOperation,
|
||||
ObjectTagConditions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard, PostObjectRequestMarker, ReqInfo,
|
||||
TABLE_DATA_PLANE_LIST_CURSOR_PREFIX, TableDataPlaneListAccess, TableDataPlaneListCursorPosition,
|
||||
TableDataPlanePublicationGuards, apply_bucket_generation_guard, apply_copy_source_bucket_generation_guard,
|
||||
authorize_copy_object_tags, authorize_internal_object_request, authorize_request, bucket_config_mutation_incarnation,
|
||||
delete_object_authorize_action, has_bypass_governance_header, load_bucket_generation_from_store,
|
||||
log_list_buckets_iam_implicit_deny, odm_read_generation, prepare_list_buckets_iam_authorization,
|
||||
prepare_odm_read_generation, recursive_force_delete_has_authenticated_caller, replication_proxy_allowed,
|
||||
replication_request_authorized, req_info_mut, req_info_ref, retained_table_data_plane_publication_guards,
|
||||
set_requested_object_lock_retain_until,
|
||||
};
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) use super::super::access::{admit_operation, resolve_req_info};
|
||||
pub(crate) use super::super::denial_audit::DenialAuditor;
|
||||
pub(crate) use super::super::helper::{AuditRequestView, DenialScope};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod owned_parity_consumer {
|
||||
pub(crate) use super::super::access::{
|
||||
ObjectTagConditions, PostObjectPolicyHeaders, PostObjectRequestMarker, ReqInfo, RequestedCopySource,
|
||||
RequestedObjectLegalHold, RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags,
|
||||
TableDataPlanePublicationGuards, authorize_request, parity_test_support,
|
||||
};
|
||||
pub(crate) use super::super::denial_audit::{DenialAuditor, DenialRateLimiter, SUPPRESSED_DENIALS_TAG};
|
||||
pub(crate) use super::super::ecfs::FS;
|
||||
pub(crate) use super::super::helper::{DenialScope, authorization_denial_entry, authorization_denial_entry_for_view};
|
||||
pub(crate) use super::super::request_context::RequestContext;
|
||||
}
|
||||
|
||||
pub(crate) mod concurrency_consumer {
|
||||
@@ -371,12 +352,16 @@ pub(crate) mod heal_control_startup_consumer {
|
||||
pub(crate) mod s3_api_consumer {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use super::super::super::s3_api::bucket::{
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_object_versions_output, build_list_objects_output,
|
||||
build_list_objects_v2_output, encode_list_objects_v2_continuation_token, parse_list_object_versions_params,
|
||||
parse_list_objects_v2_params,
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
|
||||
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
|
||||
parse_list_object_versions_params, parse_list_objects_v2_params,
|
||||
};
|
||||
|
||||
pub(crate) mod contract {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use super::super::super::super::contract::bucket::BucketInfo;
|
||||
}
|
||||
|
||||
pub(crate) mod list {
|
||||
pub(crate) use super::super::super::super::contract::list::{ListObjectVersionsInfo, ListObjectsV2Info};
|
||||
}
|
||||
|
||||
@@ -231,13 +231,6 @@ pub(crate) mod server {
|
||||
pub(crate) use crate::storage::storage_api::request_context_consumer::RequestContext;
|
||||
}
|
||||
|
||||
/// The access-phase functions the gateway stack's handlers run, shared with the legacy
|
||||
/// access hooks.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) mod access {
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{admit_operation, resolve_req_info};
|
||||
}
|
||||
|
||||
pub(crate) mod rpc {
|
||||
pub(crate) use crate::storage::storage_api::rpc_consumer::InternodeRpcService;
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -1,460 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The gateway S3 stack inside an embedded RustFS server (rustfs/backlog#2734 T2.5): one
|
||||
// operation, ListBuckets, is served end to end, and every other S3 operation is answered
|
||||
// `501 NotImplemented`. Requests are signed by the AWS SDK and travel through the whole
|
||||
// middleware stack, so what is asserted here is what a client sees.
|
||||
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use rustfs::app::context::get_global_app_context;
|
||||
use rustfs::embedded::{RustFSServer, RustFSServerBuilder, find_available_port};
|
||||
use rustfs_io_metrics::s3_http_metrics::s3_http_metrics_snapshot;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_storage_api::{BucketOperations, MakeBucketOptions};
|
||||
use std::collections::BTreeMap;
|
||||
use std::future::Future;
|
||||
|
||||
mod common;
|
||||
|
||||
const ROOT_ACCESS_KEY: &str = "gatewaystackroot";
|
||||
const ROOT_SECRET_KEY: &str = "gateway-stack-root-secret";
|
||||
const USER_SECRET_KEY: &str = "gateway-stack-user-secret";
|
||||
const ENV_S3_STACK: &str = "RUSTFS_S3_STACK";
|
||||
const ENV_KEYSTONE_ENABLE: &str = "RUSTFS_KEYSTONE_ENABLE";
|
||||
const ENV_WEBSITE_DOMAINS: &str = "RUSTFS_WEBSITE_DOMAINS";
|
||||
const ENV_KEYSTONE_AUTH_URL: &str = "RUSTFS_KEYSTONE_AUTH_URL";
|
||||
/// A Keystone endpoint nothing listens on: these tests are about start-up, not about tokens.
|
||||
const UNREACHABLE_KEYSTONE: &str = "http://127.0.0.1:1/v3";
|
||||
|
||||
fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client {
|
||||
let config = Config::builder()
|
||||
.credentials_provider(Credentials::new(access_key, secret_key, None, None, "test"))
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(endpoint)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
}
|
||||
|
||||
/// Runs `test` against an embedded server started with `RUSTFS_S3_STACK` set to `stack`.
|
||||
fn with_stack<F, Fut>(stack: Option<&'static str>, test: F)
|
||||
where
|
||||
F: FnOnce(RustFSServer) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = ()> + 'static,
|
||||
{
|
||||
with_environment(vec![(ENV_S3_STACK, stack)], test);
|
||||
}
|
||||
|
||||
/// Runs `test` against an embedded server started under `environment`.
|
||||
fn with_environment<F, Fut>(environment: Vec<(&'static str, Option<&'static str>)>, test: F)
|
||||
where
|
||||
F: FnOnce(RustFSServer) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = ()> + 'static,
|
||||
{
|
||||
temp_env::with_vars(environment, || {
|
||||
common::run_embedded_test(move || async move {
|
||||
let Some(started) = start_server().await else { return };
|
||||
test(started.expect("start embedded server")).await;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/// Starts an embedded server on a free port; `None` when the sandbox allows no port at all.
|
||||
async fn start_server() -> Option<Result<RustFSServer, String>> {
|
||||
let port = match find_available_port() {
|
||||
Ok(port) => port,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
|
||||
Err(err) => panic!("find free port: {err}"),
|
||||
};
|
||||
let started = RustFSServerBuilder::new()
|
||||
.address(format!("127.0.0.1:{port}"))
|
||||
.access_key(ROOT_ACCESS_KEY)
|
||||
.secret_key(ROOT_SECRET_KEY)
|
||||
.build()
|
||||
.await;
|
||||
// Both renderings: the refusal's text may be in either.
|
||||
Some(started.map_err(|error| format!("{error} {error:?}")))
|
||||
}
|
||||
|
||||
/// The S3 error code and HTTP status an SDK call was refused with.
|
||||
fn refusal<E: ProvideErrorMetadata>(error: &SdkError<E>) -> (Option<String>, Option<u16>) {
|
||||
let status = error.raw_response().map(|response| response.status().as_u16());
|
||||
(error.code().map(str::to_owned), status)
|
||||
}
|
||||
|
||||
/// One series of the S3 request counter: method, operation and outcome.
|
||||
type Series = (&'static str, &'static str, &'static str);
|
||||
|
||||
/// How often each series of the S3 request counter has been counted in this process.
|
||||
fn s3_request_counts() -> BTreeMap<Series, u64> {
|
||||
s3_http_metrics_snapshot()
|
||||
.into_iter()
|
||||
.map(|series| ((series.method, series.operation, series.outcome), series.total))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Every series counted since `before`, with how often. Reading the whole counter, not one
|
||||
/// series, is what makes a second count under another label visible.
|
||||
fn counted_since(before: &BTreeMap<Series, u64>) -> BTreeMap<Series, u64> {
|
||||
s3_request_counts()
|
||||
.into_iter()
|
||||
.filter_map(|(series, total)| {
|
||||
let counted = total - before.get(&series).copied().unwrap_or(0);
|
||||
(counted > 0).then_some((series, counted))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn create_bucket_in_the_store(bucket: &str) {
|
||||
get_global_app_context()
|
||||
.expect("embedded server context")
|
||||
.object_store()
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket in the store");
|
||||
}
|
||||
|
||||
async fn create_user_without_a_policy(access_key: &str) {
|
||||
get_global_app_context()
|
||||
.expect("embedded server context")
|
||||
.iam()
|
||||
.handle()
|
||||
.create_user(
|
||||
access_key,
|
||||
&rustfs_madmin::AddOrUpdateUserReq {
|
||||
secret_key: USER_SECRET_KEY.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create user without a policy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_lists_the_caller_s_buckets_and_answers_every_other_operation_501() {
|
||||
with_stack(Some("gateway"), |server| async move {
|
||||
// CreateBucket is not served by this stack yet, so the buckets are made in the store.
|
||||
create_bucket_in_the_store("gateway-stack-alpha").await;
|
||||
create_bucket_in_the_store("gateway-stack-beta").await;
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
|
||||
let listing = client.list_buckets().send().await.expect("signed ListBuckets");
|
||||
let mut names: Vec<&str> = listing.buckets().iter().filter_map(|bucket| bucket.name()).collect();
|
||||
names.sort_unstable();
|
||||
assert_eq!(names, ["gateway-stack-alpha", "gateway-stack-beta"]);
|
||||
assert!(listing.owner().and_then(|owner| owner.id()).is_some(), "the listing names its owner");
|
||||
assert!(
|
||||
listing.buckets().iter().all(|bucket| bucket.creation_date().is_some()),
|
||||
"every bucket carries its creation date"
|
||||
);
|
||||
|
||||
let get = client
|
||||
.get_object()
|
||||
.bucket("gateway-stack-alpha")
|
||||
.key("object")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("GetObject is not registered");
|
||||
assert_eq!(refusal(&get), (Some("NotImplemented".to_owned()), Some(501)));
|
||||
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket("gateway-stack-alpha")
|
||||
.key("object")
|
||||
.body(ByteStream::from_static(b"payload"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("PutObject is not registered");
|
||||
assert_eq!(refusal(&put), (Some("NotImplemented".to_owned()), Some(501)));
|
||||
|
||||
let create = client
|
||||
.create_bucket()
|
||||
.bucket("gateway-stack-gamma")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("CreateBucket is not registered");
|
||||
assert_eq!(refusal(&create), (Some("NotImplemented".to_owned()), Some(501)));
|
||||
|
||||
let listing = client.list_buckets().send().await.expect("ListBuckets after the refusals");
|
||||
assert_eq!(listing.buckets().len(), 2, "a refused write created nothing");
|
||||
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_refuses_callers_it_cannot_admit() {
|
||||
with_stack(Some("gateway"), |server| async move {
|
||||
create_bucket_in_the_store("gateway-stack-alpha").await;
|
||||
create_user_without_a_policy("gatewaystacknopolicy").await;
|
||||
let endpoint = server.endpoint();
|
||||
|
||||
// Anonymous: no credentials at all.
|
||||
let anonymous = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("http client")
|
||||
.get(format!("{endpoint}/"))
|
||||
.send()
|
||||
.await
|
||||
.expect("anonymous ListBuckets is answered");
|
||||
assert_eq!(anonymous.status().as_u16(), 403);
|
||||
let request_id = anonymous
|
||||
.headers()
|
||||
.get("x-amz-request-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned)
|
||||
.expect("the refusal names its request");
|
||||
assert_eq!(
|
||||
anonymous.headers().get("x-request-id").and_then(|value| value.to_str().ok()),
|
||||
Some(request_id.as_str()),
|
||||
"one identifier names the request in both headers"
|
||||
);
|
||||
let body = anonymous.text().await.expect("refusal body");
|
||||
assert!(body.contains("<Code>AccessDenied</Code>"), "{body}");
|
||||
|
||||
// The right access key with the wrong secret.
|
||||
let wrong_secret = s3_client(&endpoint, ROOT_ACCESS_KEY, "not-the-root-secret")
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a wrong secret signs nothing");
|
||||
assert_eq!(refusal(&wrong_secret), (Some("SignatureDoesNotMatch".to_owned()), Some(403)));
|
||||
|
||||
// An access key nobody issued: the caller's fault, not the server's.
|
||||
let unknown_key = s3_client(&endpoint, "gatewaystackunknown", USER_SECRET_KEY)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("an unknown access key is refused");
|
||||
assert_eq!(refusal(&unknown_key), (Some("InvalidAccessKeyId".to_owned()), Some(403)));
|
||||
|
||||
// A real user whose policy grants nothing: it may neither list every bucket nor see one.
|
||||
let no_permission = s3_client(&endpoint, "gatewaystacknopolicy", USER_SECRET_KEY)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a caller without permission lists nothing");
|
||||
assert_eq!(refusal(&no_permission), (Some("AccessDenied".to_owned()), Some(403)));
|
||||
|
||||
// The root caller is still served by the same server.
|
||||
let listing = s3_client(&endpoint, server.access_key(), server.secret_key())
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect("signed ListBuckets");
|
||||
assert_eq!(listing.buckets().len(), 1);
|
||||
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
/// A website-host request reaches the legacy website route on the gateway stack too
|
||||
/// (rustfs/backlog#2734 T2.11, R-064). Before that route existed here, `GET /` on a website host
|
||||
/// was a path-style ListBuckets, which the gateway refuses anonymously with `AccessDenied`. The
|
||||
/// full website differential against the legacy stack is `e2e_test::website_stack_interim_test`.
|
||||
#[test]
|
||||
fn the_gateway_stack_leaves_a_website_host_to_the_legacy_website_route() {
|
||||
let environment = vec![(ENV_S3_STACK, Some("gateway")), (ENV_WEBSITE_DOMAINS, Some("website.test"))];
|
||||
with_environment(environment, |server| async move {
|
||||
// A bucket with no website configuration: legacy's website route answers its plain page.
|
||||
create_bucket_in_the_store("gateway-stack-site").await;
|
||||
let endpoint = server.endpoint();
|
||||
let http = reqwest::Client::builder().no_proxy().build().expect("http client");
|
||||
|
||||
for host in [
|
||||
"gateway-stack-site.website.test",
|
||||
"website.test",
|
||||
"Gateway-Stack-Site.Website.Test.",
|
||||
] {
|
||||
let website = http
|
||||
.get(format!("{endpoint}/"))
|
||||
.header(reqwest::header::HOST, host)
|
||||
.send()
|
||||
.await
|
||||
.expect("a website-host request is answered");
|
||||
assert_eq!(website.status().as_u16(), 404, "{host}");
|
||||
assert_eq!(
|
||||
website
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("text/html; charset=utf-8"),
|
||||
"{host}"
|
||||
);
|
||||
assert_eq!(
|
||||
website.text().await.expect("website body"),
|
||||
"<html><body>Not Found</body></html>",
|
||||
"{host}"
|
||||
);
|
||||
}
|
||||
|
||||
// The same request on a host that is no website domain is still the gateway's ListBuckets.
|
||||
for host in ["gateway-stack-site.elsewhere.test", "evilwebsite.test"] {
|
||||
let not_website = http
|
||||
.get(format!("{endpoint}/"))
|
||||
.header(reqwest::header::HOST, host)
|
||||
.send()
|
||||
.await
|
||||
.expect("a request on another host is answered");
|
||||
assert_eq!(not_website.status().as_u16(), 403, "{host}");
|
||||
let body = not_website.text().await.expect("refusal body");
|
||||
assert!(body.contains("<Code>AccessDenied</Code>"), "{host}: {body}");
|
||||
}
|
||||
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
/// The S3 request counter belongs to RustFS's layer in front of both stacks, and the operation
|
||||
/// label is set by the code a handler calls. That only works if the gateway polls its handler
|
||||
/// inside that layer's scope, on the request's own task, which is what this observes.
|
||||
#[test]
|
||||
fn the_gateway_stack_counts_each_request_once_under_the_label_the_legacy_stack_uses() {
|
||||
with_stack(Some("gateway"), |server| async move {
|
||||
let endpoint = server.endpoint();
|
||||
|
||||
let before = s3_request_counts();
|
||||
s3_client(&endpoint, server.access_key(), server.secret_key())
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect("signed ListBuckets");
|
||||
assert_eq!(
|
||||
counted_since(&before),
|
||||
BTreeMap::from([(("GET", S3Operation::ListBuckets.as_str(), "2xx"), 1)]),
|
||||
"one served request is counted once, as ListBuckets"
|
||||
);
|
||||
|
||||
let before = s3_request_counts();
|
||||
let refused = s3_client(&endpoint, ROOT_ACCESS_KEY, "not-the-root-secret")
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a wrong secret signs nothing");
|
||||
assert_eq!(refusal(&refused), (Some("SignatureDoesNotMatch".to_owned()), Some(403)));
|
||||
assert_eq!(
|
||||
counted_since(&before),
|
||||
BTreeMap::from([(("GET", "unknown", "4xx"), 1)]),
|
||||
"a request refused by authentication is counted once, under no operation"
|
||||
);
|
||||
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
/// The pinned gateway admits a request that carries only `X-Auth-Token` as anonymous, while the
|
||||
/// RustFS code behind it reads the Keystone identity. Until the gateway authenticates the token
|
||||
/// (rustfs/gateway#1586) the two settings together refuse to start, and nothing is served.
|
||||
#[test]
|
||||
fn the_gateway_stack_refuses_to_start_with_keystone_enabled() {
|
||||
let environment = [
|
||||
(ENV_S3_STACK, Some("gateway")),
|
||||
(ENV_KEYSTONE_ENABLE, Some("true")),
|
||||
(ENV_KEYSTONE_AUTH_URL, Some(UNREACHABLE_KEYSTONE)),
|
||||
];
|
||||
temp_env::with_vars(environment, || {
|
||||
common::run_embedded_test(|| async {
|
||||
let Some(started) = start_server().await else { return };
|
||||
let refusal = match started {
|
||||
Ok(server) => {
|
||||
server.shutdown().await;
|
||||
panic!("the gateway stack started with Keystone enabled");
|
||||
}
|
||||
Err(refusal) => refusal,
|
||||
};
|
||||
for named in ["RUSTFS_S3_STACK=gateway", "RUSTFS_KEYSTONE_ENABLE", "rustfs/gateway#1586"] {
|
||||
assert!(refusal.contains(named), "{named} is missing from: {refusal}");
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_stack_starts_with_keystone_enabled() {
|
||||
let environment = vec![
|
||||
(ENV_S3_STACK, None),
|
||||
(ENV_KEYSTONE_ENABLE, Some("true")),
|
||||
(ENV_KEYSTONE_AUTH_URL, Some(UNREACHABLE_KEYSTONE)),
|
||||
];
|
||||
with_environment(environment, |server| async move {
|
||||
let listing = s3_client(&server.endpoint(), server.access_key(), server.secret_key())
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect("signed ListBuckets on the legacy stack with Keystone enabled");
|
||||
assert!(listing.buckets().is_empty());
|
||||
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_stack_still_serves_every_operation() {
|
||||
with_stack(None, |server| async move {
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket("legacy-stack-alpha")
|
||||
.send()
|
||||
.await
|
||||
.expect("CreateBucket on the legacy stack");
|
||||
client
|
||||
.put_object()
|
||||
.bucket("legacy-stack-alpha")
|
||||
.key("object")
|
||||
.body(ByteStream::from_static(b"payload"))
|
||||
.send()
|
||||
.await
|
||||
.expect("PutObject on the legacy stack");
|
||||
let before = s3_request_counts();
|
||||
let listing = client.list_buckets().send().await.expect("ListBuckets on the legacy stack");
|
||||
assert_eq!(
|
||||
counted_since(&before),
|
||||
BTreeMap::from([(("GET", S3Operation::ListBuckets.as_str(), "2xx"), 1)]),
|
||||
"the label the gateway stack is held to is the one this stack counts under"
|
||||
);
|
||||
let names: Vec<&str> = listing.buckets().iter().filter_map(|bucket| bucket.name()).collect();
|
||||
assert_eq!(names, ["legacy-stack-alpha"]);
|
||||
assert!(listing.owner().and_then(|owner| owner.id()).is_some(), "the listing names its owner");
|
||||
|
||||
let missing = client
|
||||
.get_object()
|
||||
.bucket("legacy-stack-alpha")
|
||||
.key("missing")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a missing key");
|
||||
assert_eq!(refusal(&missing), (Some("NoSuchKey".to_owned()), Some(404)));
|
||||
|
||||
create_user_without_a_policy("legacystacknopolicy").await;
|
||||
let no_permission = s3_client(&server.endpoint(), "legacystacknopolicy", USER_SECRET_KEY)
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a caller without permission lists nothing on either stack");
|
||||
assert_eq!(refusal(&no_permission), (Some("AccessDenied".to_owned()), Some(403)));
|
||||
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
+1
-1
@@ -169,7 +169,7 @@ their issue closes.
|
||||
|---|---|---|---|
|
||||
| `codemods/` | dev-tool | Scripted, idempotent source migrations for the gateway migration (rustfs/backlog#2734); `s3_error_to_s3_types.py` moves s3s error-type paths onto `rustfs-s3-types` per crate closure and refuses the s3s edge allowlist | `make script-tests` (`--self-test`); usage in each script's docstring |
|
||||
| `fuzz/` | ci-gate | Unified cargo-fuzz runner and helpers for the `fuzz/` sub-workspace | fuzz.yml; `fuzz/README.md` |
|
||||
| `release/` | ci-gate | Release creation and DEB/RPM version-normalization helpers | build.yml; package.yml |
|
||||
| `release/` | ci-gate | Release creation and DEB/RPM version-normalization helpers | build.yml; package.yml; manual-main-packages.yml |
|
||||
| `s3-tests/` | ci-gate | ceph/s3-tests compatibility harness (allow-lists, patches, report tooling) | ci.yml; e2e-s3tests.yml; `scripts/s3-tests/README.md` |
|
||||
| `security/` | ci-gate | Workflow-pin enforcement and release supply-chain asset generation | audit.yml; build.yml |
|
||||
| `table-catalog/` | dev-tool | S3-Tables / pyiceberg validation suite | `docs/architecture/s3-tables-support-matrix.md` |
|
||||
|
||||
@@ -1,226 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Keeps the `gateway-stack` cargo feature of the rustfs crate out of every release build while the
|
||||
# gateway S3 stack is under construction (rustfs/backlog#2734 T2.5).
|
||||
#
|
||||
# Usage: scripts/check_gateway_stack_release_boundary.sh [--root DIR]
|
||||
# scripts/check_gateway_stack_release_boundary.sh --self-test
|
||||
#
|
||||
# Three facts are checked, none of which needs Cargo:
|
||||
# 1. rustfs/src/server/gateway_stack.rs refuses to compile without debug assertions, so a
|
||||
# release-profile build that enables the feature fails in the compiler;
|
||||
# 2. neither the `default` nor the `full` feature list of rustfs/Cargo.toml enables it, directly
|
||||
# or through a feature that enables it, and the feature itself still exists (a guard whose
|
||||
# subject is gone must fail, not skip);
|
||||
# 3. no file that produces a release artifact names the feature or one that enables it.
|
||||
# A missing input is an error: every file below exists in this repository.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
FEATURE="gateway-stack"
|
||||
GUARD_SOURCE="rustfs/src/server/gateway_stack.rs"
|
||||
GUARD_CFG='#[cfg(not(debug_assertions))]'
|
||||
GUARD_ERROR='compile_error!("gateway-stack is restricted to debug builds");'
|
||||
MANIFEST="rustfs/Cargo.toml"
|
||||
# Everything that builds or packages a release binary.
|
||||
RELEASE_FILES=(
|
||||
".github/workflows/build.yml"
|
||||
".github/workflows/docker.yml"
|
||||
"build-rustfs.sh"
|
||||
"docker-buildx.sh"
|
||||
"Dockerfile"
|
||||
"Dockerfile.glibc"
|
||||
"Dockerfile.source"
|
||||
"rustfs.spec"
|
||||
"Justfile"
|
||||
".config/make/build.mak"
|
||||
".config/make/build-docker.mak"
|
||||
)
|
||||
|
||||
fail() {
|
||||
echo "gateway-stack release boundary: $*" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# Prints the value of one `name = [...]` feature line of the manifest.
|
||||
feature_list() {
|
||||
local manifest="$1" name="$2"
|
||||
grep -E "^${name}[[:space:]]*=" "$manifest" || true
|
||||
}
|
||||
|
||||
# Prints the feature and every feature that enables it, directly or through another, one per line.
|
||||
enabling_features() {
|
||||
local manifest="$1" known="$FEATURE" grown name
|
||||
while true; do
|
||||
grown="$known"
|
||||
while IFS= read -r name; do
|
||||
grep -qxF -- "$name" <<<"$grown" || grown+=$'\n'"$name"
|
||||
done < <(
|
||||
while IFS= read -r name; do
|
||||
grep -E "^[A-Za-z0-9_-]+[[:space:]]*=.*\"${name}\"" "$manifest" | sed -E 's/[[:space:]]*=.*//' || true
|
||||
done <<<"$known"
|
||||
)
|
||||
[[ "$grown" == "$known" ]] && break
|
||||
known="$grown"
|
||||
done
|
||||
printf '%s\n' "$known"
|
||||
}
|
||||
|
||||
check_root() {
|
||||
local root="$1" status=0 file line name enabler
|
||||
local -a enablers=("$FEATURE")
|
||||
|
||||
file="${root}/${GUARD_SOURCE}"
|
||||
if [[ ! -f "$file" ]]; then
|
||||
fail "${GUARD_SOURCE} is missing" || status=1
|
||||
else
|
||||
# The cfg must sit on the line directly above the compile_error.
|
||||
line="$(grep -nF "$GUARD_ERROR" "$file" | head -n 1 | cut -d: -f1)"
|
||||
if [[ -z "$line" ]]; then
|
||||
fail "${GUARD_SOURCE} no longer refuses a build without debug assertions" || status=1
|
||||
elif [[ "$(sed -n "$((line - 1))p" "$file" | tr -d '[:space:]')" != "$(tr -d '[:space:]' <<<"$GUARD_CFG")" ]]; then
|
||||
fail "the compile_error in ${GUARD_SOURCE} is not gated on cfg(not(debug_assertions))" || status=1
|
||||
fi
|
||||
fi
|
||||
|
||||
file="${root}/${MANIFEST}"
|
||||
if [[ ! -f "$file" ]]; then
|
||||
fail "${MANIFEST} is missing" || status=1
|
||||
else
|
||||
if [[ -z "$(feature_list "$file" "$FEATURE")" ]]; then
|
||||
fail "${MANIFEST} no longer declares the ${FEATURE} feature this guard protects" || status=1
|
||||
fi
|
||||
mapfile -t enablers < <(enabling_features "$file")
|
||||
for name in default full; do
|
||||
line="$(feature_list "$file" "$name")"
|
||||
if [[ -z "$line" ]]; then
|
||||
fail "${MANIFEST} has no ${name} feature list" || status=1
|
||||
fi
|
||||
for enabler in "${enablers[@]}"; do
|
||||
if [[ "$enabler" == "$name" ]]; then
|
||||
fail "the ${name} feature list of ${MANIFEST} enables ${FEATURE}" || status=1
|
||||
fi
|
||||
done
|
||||
done
|
||||
fi
|
||||
|
||||
for file in "${RELEASE_FILES[@]}"; do
|
||||
if [[ ! -f "${root}/${file}" ]]; then
|
||||
fail "release file ${file} is missing; update RELEASE_FILES" || status=1
|
||||
continue
|
||||
fi
|
||||
for enabler in "${enablers[@]}"; do
|
||||
if grep -qF -- "$enabler" "${root}/${file}"; then
|
||||
fail "${file} names ${enabler}, which enables ${FEATURE}; release artifacts must not" || status=1
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
return "$status"
|
||||
}
|
||||
|
||||
self_test() {
|
||||
local scratch fixture failures=0
|
||||
scratch="$(mktemp -d)"
|
||||
|
||||
make_fixture() {
|
||||
fixture="${scratch}/$1"
|
||||
mkdir -p "${fixture}/rustfs/src/server" "${fixture}/.github/workflows" "${fixture}/.config/make"
|
||||
printf '%s\n%s\n' "$GUARD_CFG" "$GUARD_ERROR" >"${fixture}/${GUARD_SOURCE}"
|
||||
printf '%s\n' '[features]' 'default = ["ftps"]' 'full = ["ftps", "swift"]' 'gateway-stack = ["dep:rustfs-gateway"]' \
|
||||
>"${fixture}/${MANIFEST}"
|
||||
for file in "${RELEASE_FILES[@]}"; do
|
||||
printf 'cargo build --release -p rustfs\n' >"${fixture}/${file}"
|
||||
done
|
||||
}
|
||||
|
||||
expect() {
|
||||
local wanted="$1" label="$2"
|
||||
if check_root "$fixture" >/dev/null 2>&1; then
|
||||
[[ "$wanted" == "pass" ]] || { echo "self-test: ${label} was accepted" >&2; failures=$((failures + 1)); }
|
||||
else
|
||||
[[ "$wanted" == "fail" ]] || { echo "self-test: ${label} was refused" >&2; failures=$((failures + 1)); }
|
||||
fi
|
||||
}
|
||||
|
||||
make_fixture valid
|
||||
expect pass "a repository that keeps the boundary"
|
||||
|
||||
make_fixture no-compile-error
|
||||
printf '// nothing here\n' >"${fixture}/${GUARD_SOURCE}"
|
||||
expect fail "a gateway stack without the compile-time refusal"
|
||||
|
||||
make_fixture ungated-compile-error
|
||||
printf '%s\n%s\n' '#[cfg(test)]' "$GUARD_ERROR" >"${fixture}/${GUARD_SOURCE}"
|
||||
expect fail "a compile_error gated on something else"
|
||||
|
||||
make_fixture missing-source
|
||||
rm "${fixture}/${GUARD_SOURCE}"
|
||||
expect fail "a missing guard source"
|
||||
|
||||
make_fixture default-enables
|
||||
sed -i.bak 's/^default = .*/default = ["ftps", "gateway-stack"]/' "${fixture}/${MANIFEST}"
|
||||
expect fail "a default feature list that enables the stack"
|
||||
|
||||
make_fixture full-enables
|
||||
sed -i.bak 's/^full = .*/full = ["ftps", "gateway-stack"]/' "${fixture}/${MANIFEST}"
|
||||
expect fail "a full feature list that enables the stack"
|
||||
|
||||
make_fixture feature-gone
|
||||
sed -i.bak '/^gateway-stack = /d' "${fixture}/${MANIFEST}"
|
||||
expect fail "a manifest without the feature"
|
||||
|
||||
make_fixture full-enables-through-another
|
||||
printf '%s\n' 'hooks = ["gateway-stack"]' >>"${fixture}/${MANIFEST}"
|
||||
sed -i.bak 's/^full = .*/full = ["ftps", "hooks"]/' "${fixture}/${MANIFEST}"
|
||||
expect fail "a full feature list that enables the stack through another feature"
|
||||
|
||||
make_fixture debug-only-enabler
|
||||
printf '%s\n' 'hooks = ["gateway-stack"]' >>"${fixture}/${MANIFEST}"
|
||||
expect pass "a feature outside default and full that enables the stack"
|
||||
|
||||
make_fixture release-file-names-enabler
|
||||
printf '%s\n' 'hooks = ["gateway-stack"]' >>"${fixture}/${MANIFEST}"
|
||||
printf 'cargo build --release -p rustfs --features hooks\n' >"${fixture}/build-rustfs.sh"
|
||||
expect fail "a release file that enables the stack through another feature"
|
||||
|
||||
make_fixture release-workflow
|
||||
printf 'cargo build --release -p rustfs --features gateway-stack\n' >"${fixture}/.github/workflows/build.yml"
|
||||
expect fail "a release workflow that enables the stack"
|
||||
|
||||
make_fixture missing-release-file
|
||||
rm "${fixture}/build-rustfs.sh"
|
||||
expect fail "a missing release file"
|
||||
|
||||
rm -rf "$scratch"
|
||||
if ((failures > 0)); then
|
||||
echo "gateway-stack release boundary self-test: ${failures} failure(s)" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "gateway-stack release boundary self-test passed"
|
||||
}
|
||||
|
||||
main() {
|
||||
local root
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
case "${1:-}" in
|
||||
--self-test)
|
||||
[[ $# -eq 1 ]] || { echo "usage: $0 [--root DIR] | --self-test" >&2; exit 2; }
|
||||
self_test
|
||||
;;
|
||||
--root)
|
||||
[[ $# -eq 2 ]] || { echo "usage: $0 [--root DIR] | --self-test" >&2; exit 2; }
|
||||
check_root "$2"
|
||||
echo "gateway-stack release boundary holds"
|
||||
;;
|
||||
"")
|
||||
check_root "$root"
|
||||
echo "gateway-stack release boundary holds"
|
||||
;;
|
||||
*)
|
||||
echo "usage: $0 [--root DIR] | --self-test" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+2
-4
@@ -17,7 +17,7 @@ ROOT = Path(__file__).resolve().parent.parent
|
||||
ALWAYS_JOBS = ("classify-changes", "typos", "quick-checks")
|
||||
CODE_JOBS = (
|
||||
"workspace-lint-and-build", "workspace-tests", "core-contract", "connect-feature-boundaries", "fips-provider", "test-and-lint-rio-v2",
|
||||
"test-and-lint-protocols", "test-and-lint-gateway-stack", "build-rustfs-debug-binary", "uring-integration",
|
||||
"test-and-lint-protocols", "build-rustfs-debug-binary", "uring-integration",
|
||||
"e2e-tests", "s3-implemented-tests", "s3-lifecycle-behavior-tests",
|
||||
)
|
||||
# Specialized boundary, hardware and alternate-feature lanes do not gate a
|
||||
@@ -33,7 +33,7 @@ NIGHTLY_JOBS = ("connect-feature-boundaries", "fips-provider", "uring-integratio
|
||||
# against this revision's expected job contract. The workflow refuses a plan
|
||||
# that does not carry the current marker and falls back to the full matrix, so
|
||||
# a base planner predating the contract cannot authorize a narrower selection.
|
||||
PLAN_CONTRACT = "4"
|
||||
PLAN_CONTRACT = "3"
|
||||
# These jobs still exercise the server and its black-box harness when only E2E
|
||||
# Rust sources change. Production code, manifests and shared test configuration
|
||||
# always select the full matrix. No production package depends on e2e_test.
|
||||
@@ -299,8 +299,6 @@ def configurations(jobs: dict[str, str]) -> list[dict]:
|
||||
("protocol-sftp-tests", "test-and-lint-protocols", "test", ["sftp"]),
|
||||
("protocol-http3-lint", "test-and-lint-protocols", "dev", ["http3", "swift"]),
|
||||
("protocol-http3-tests", "test-and-lint-protocols", "test", ["http3", "swift"]),
|
||||
("gateway-stack-lint", "test-and-lint-gateway-stack", "dev", ["gateway-stack"]),
|
||||
("gateway-stack-tests", "test-and-lint-gateway-stack", "test", ["gateway-stack"]),
|
||||
("server-rio-v2", "build-rustfs-debug-binary-rio-v2", "dev", ["e2e-test-hooks", "rio-v2"]),
|
||||
]
|
||||
return [{"id": identity, "job": job, "target": "x86_64-unknown-linux-gnu", "profile": profile,
|
||||
|
||||
@@ -13,7 +13,7 @@ OUTPUT="$4"
|
||||
|
||||
case "$ARCH" in
|
||||
amd64|arm64) ;;
|
||||
*) echo "Unsupported rc architecture: $ARCH" >&2; exit 1 ;;
|
||||
*) echo "Unsupported rc architecture: '$ARCH' (expected amd64 or arm64; an empty value means the caller's TARGETARCH build argument was not set)" >&2; exit 1 ;;
|
||||
esac
|
||||
case "$LIBC" in
|
||||
musl) LIBC_SUFFIX="" ;;
|
||||
|
||||
Executable
+406
@@ -0,0 +1,406 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Manual zip/deb/rpm packaging for one Linux target.
|
||||
# Version rules and deb/rpm metadata come from package_versions.sh.
|
||||
# deb/rpm payloads are built by build_deb.sh and build_rpm.sh.
|
||||
# Dependency strings match scripts/release/deliver.py package().
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
helper="${repo_root}/scripts/release/package_versions.sh"
|
||||
|
||||
die() {
|
||||
printf 'package_manual_artifacts: %s\n' "$1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
usage() {
|
||||
cat <<'EOF' >&2
|
||||
usage: package_manual_artifacts.sh matrix <x86|arm|all>
|
||||
package_manual_artifacts.sh validate --version VER --arch ARCH --libc LIBC
|
||||
package_manual_artifacts.sh package --version VER --arch ARCH --libc LIBC --binary PATH --output DIR [--zip-extra FILE]...
|
||||
EOF
|
||||
exit 2
|
||||
}
|
||||
|
||||
require_value() {
|
||||
local flag=$1
|
||||
# An empty value is a version/path error, not a missing argument. Flag-shaped
|
||||
# values mean the next option was consumed by mistake.
|
||||
if [[ $# -lt 2 || ${2:-} == --* ]]; then
|
||||
die "missing value for ${flag}"
|
||||
fi
|
||||
}
|
||||
|
||||
resolve_version() {
|
||||
local version=$1 arch=$2 libc=$3
|
||||
local release_err prerelease_err release_out prerelease_out
|
||||
release_err=$(mktemp)
|
||||
prerelease_err=$(mktemp)
|
||||
if release_out=$(bash "$helper" release "$version" "" "$arch" "$libc" 2>"$release_err"); then
|
||||
printf 'release\n%s\n' "$release_out"
|
||||
rm -f "$release_err" "$prerelease_err"
|
||||
return 0
|
||||
fi
|
||||
if prerelease_out=$(bash "$helper" prerelease "$version" "" "$arch" "$libc" 2>"$prerelease_err"); then
|
||||
printf 'prerelease\n%s\n' "$prerelease_out"
|
||||
rm -f "$release_err" "$prerelease_err"
|
||||
return 0
|
||||
fi
|
||||
printf 'invalid package version: %s\n' "$version" >&2
|
||||
printf 'not a release: %s\n' "$(<"$release_err")" >&2
|
||||
printf 'not a prerelease: %s\n' "$(<"$prerelease_err")" >&2
|
||||
rm -f "$release_err" "$prerelease_err"
|
||||
return 1
|
||||
}
|
||||
|
||||
load_versions() {
|
||||
local resolved line key value
|
||||
resolved=$(resolve_version "$version" "$arch" "$libc") || exit 1
|
||||
build_type=${resolved%%$'\n'*}
|
||||
local rest=${resolved#*$'\n'}
|
||||
deb_version=''
|
||||
rpm_version=''
|
||||
rpm_release=''
|
||||
deb_file=''
|
||||
rpm_file=''
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
[[ -n "$line" ]] || continue
|
||||
key=${line%%=*}
|
||||
value=${line#*=}
|
||||
case "$key" in
|
||||
deb_version) deb_version=$value ;;
|
||||
rpm_version) rpm_version=$value ;;
|
||||
rpm_release) rpm_release=$value ;;
|
||||
deb_file) deb_file=$value ;;
|
||||
rpm_file) rpm_file=$value ;;
|
||||
*) die "unexpected package_versions field: ${key}" ;;
|
||||
esac
|
||||
done <<< "$rest"
|
||||
[[ -n "$deb_version" && -n "$rpm_version" && -n "$rpm_release" && -n "$deb_file" && -n "$rpm_file" ]] ||
|
||||
die "incomplete package version output"
|
||||
local stem=${deb_file%.deb}
|
||||
[[ "$stem" == "${rpm_file%.rpm}" ]] || die "deb and rpm file names use different stems"
|
||||
[[ "$stem" == "rustfs-linux-${arch}-${libc}-v${version}" ]] ||
|
||||
die "package stem does not match rustfs-linux-${arch}-${libc}-v${version}"
|
||||
zip_file="${stem}.zip"
|
||||
case "$build_type" in
|
||||
release | prerelease) ;;
|
||||
*) die "unsupported build type: ${build_type}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
emit_resolved() {
|
||||
printf 'build_type=%s\n' "$build_type"
|
||||
printf 'deb_version=%s\n' "$deb_version"
|
||||
printf 'rpm_version=%s\n' "$rpm_version"
|
||||
printf 'rpm_release=%s\n' "$rpm_release"
|
||||
printf 'deb_file=%s\n' "$deb_file"
|
||||
printf 'rpm_file=%s\n' "$rpm_file"
|
||||
printf 'zip_file=%s\n' "$zip_file"
|
||||
}
|
||||
|
||||
emit_matrix() {
|
||||
local choice=$1
|
||||
command -v python3 >/dev/null 2>&1 || die "python3 is required"
|
||||
python3 - "$choice" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
choice = sys.argv[1]
|
||||
# Linux rows match the release matrix in .github/workflows/build.yml.
|
||||
# arch, libc, and build_os are the packaging names for those same rows.
|
||||
rows = [
|
||||
{"target_id": "linux-x86_64-musl", "os": "sm-standard-4", "target": "x86_64-unknown-linux-musl", "cross": False, "platform": "linux", "rustflags": "", "features": "", "glibc": "", "arch": "x86_64", "libc": "musl", "build_os": "linux-x86_64"},
|
||||
{"target_id": "linux-aarch64-musl", "os": "ubicloud-standard-16-arm", "target": "aarch64-unknown-linux-musl", "cross": False, "platform": "linux", "rustflags": "", "features": "", "glibc": "", "arch": "aarch64", "libc": "musl", "build_os": "linux-aarch64"},
|
||||
{"target_id": "linux-x86_64-gnu", "os": "sm-standard-4", "target": "x86_64-unknown-linux-gnu", "cross": True, "platform": "linux", "rustflags": "", "features": "pyroscope", "glibc": "2.31", "arch": "x86_64", "libc": "gnu", "build_os": "linux-x86_64"},
|
||||
{"target_id": "linux-aarch64-gnu", "os": "ubicloud-standard-16-arm", "target": "aarch64-unknown-linux-gnu", "cross": True, "platform": "linux", "rustflags": "", "features": "pyroscope", "glibc": "2.31", "arch": "aarch64", "libc": "gnu", "build_os": "linux-aarch64"},
|
||||
]
|
||||
if choice == "all":
|
||||
selected = rows
|
||||
elif choice == "x86":
|
||||
selected = [row for row in rows if row["arch"] == "x86_64"]
|
||||
elif choice == "arm":
|
||||
selected = [row for row in rows if row["arch"] == "aarch64"]
|
||||
else:
|
||||
raise SystemExit(
|
||||
"package_manual_artifacts: unsupported architecture "
|
||||
f"(expected x86, arm, or all): {choice}"
|
||||
)
|
||||
if len(selected) == 0:
|
||||
raise SystemExit("package_manual_artifacts: no targets selected")
|
||||
json.dump({"include": selected}, sys.stdout, separators=(",", ":"))
|
||||
sys.stdout.write("\n")
|
||||
PY
|
||||
}
|
||||
|
||||
parse_identity() {
|
||||
version=''
|
||||
arch=''
|
||||
libc=''
|
||||
local saw_version=0 saw_arch=0 saw_libc=0
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--version)
|
||||
require_value "$@"
|
||||
saw_version=1
|
||||
version=$2
|
||||
shift 2
|
||||
;;
|
||||
--arch)
|
||||
require_value "$@"
|
||||
saw_arch=1
|
||||
arch=$2
|
||||
shift 2
|
||||
;;
|
||||
--libc)
|
||||
require_value "$@"
|
||||
saw_libc=1
|
||||
libc=$2
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
die "unknown argument: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
[[ "$saw_version" == 1 && "$saw_arch" == 1 && "$saw_libc" == 1 ]] ||
|
||||
die "validate requires --version, --arch, and --libc"
|
||||
}
|
||||
|
||||
abspath() {
|
||||
local path=$1
|
||||
local dir base
|
||||
dir=$(cd "$(dirname "$path")" && pwd)
|
||||
base=$(basename "$path")
|
||||
printf '%s/%s' "$dir" "$base"
|
||||
}
|
||||
|
||||
sha256_file() {
|
||||
sha256sum "$1" | awk 'NR==1 { print $1 }'
|
||||
}
|
||||
|
||||
package_binary() {
|
||||
local binary='' output=''
|
||||
local -a zip_extras=()
|
||||
local saw_version=0 saw_arch=0 saw_libc=0
|
||||
version=''
|
||||
arch=''
|
||||
libc=''
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--version)
|
||||
require_value "$@"
|
||||
saw_version=1
|
||||
version=$2
|
||||
shift 2
|
||||
;;
|
||||
--arch)
|
||||
require_value "$@"
|
||||
saw_arch=1
|
||||
arch=$2
|
||||
shift 2
|
||||
;;
|
||||
--libc)
|
||||
require_value "$@"
|
||||
saw_libc=1
|
||||
libc=$2
|
||||
shift 2
|
||||
;;
|
||||
--binary)
|
||||
require_value "$@"
|
||||
binary=$2
|
||||
shift 2
|
||||
;;
|
||||
--output)
|
||||
require_value "$@"
|
||||
output=$2
|
||||
shift 2
|
||||
;;
|
||||
--zip-extra)
|
||||
require_value "$@"
|
||||
zip_extras+=("$2")
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
die "unknown argument: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
[[ "$saw_version" == 1 && "$saw_arch" == 1 && "$saw_libc" == 1 && -n "$binary" && -n "$output" ]] ||
|
||||
die "package requires --version, --arch, --libc, --binary, and --output"
|
||||
[[ -f "$binary" && ! -L "$binary" ]] || die "binary must be a regular file"
|
||||
load_versions
|
||||
|
||||
local -a extra_names=()
|
||||
local extra base seen=' '
|
||||
local has_json=0 has_sha=0
|
||||
for extra in "${zip_extras[@]}"; do
|
||||
base=$(basename "$extra")
|
||||
case "$base" in
|
||||
rustfs.cpu-symbol-catalog.json) has_json=1 ;;
|
||||
rustfs.cpu-symbol-catalog.sha256) has_sha=1 ;;
|
||||
*) die "unsupported zip member: ${base}" ;;
|
||||
esac
|
||||
[[ "$seen" != *" ${base} "* ]] || die "duplicate zip member: ${base}"
|
||||
seen+=" ${base} "
|
||||
[[ -f "$extra" && ! -L "$extra" ]] || die "zip extra must be a regular file: ${base}"
|
||||
extra_names+=("$base")
|
||||
done
|
||||
[[ "$has_json" == "$has_sha" ]] ||
|
||||
die "cpu symbol catalog json and sha256 must be packaged together"
|
||||
|
||||
local tool
|
||||
for tool in zip unzip sha256sum dpkg-deb fakeroot rpm rpm2cpio cpio fpm; do
|
||||
command -v "$tool" >/dev/null 2>&1 || die "required tool not found: ${tool}"
|
||||
done
|
||||
|
||||
binary=$(abspath "$binary")
|
||||
local -a absolute_extras=()
|
||||
for extra in "${zip_extras[@]}"; do
|
||||
absolute_extras+=("$(abspath "$extra")")
|
||||
done
|
||||
mkdir -p "$output"
|
||||
output=$(cd "$output" && pwd)
|
||||
local produced
|
||||
for produced in "$zip_file" "$deb_file" "$rpm_file"; do
|
||||
[[ ! -e "$output/$produced" ]] || die "refusing to overwrite ${output}/${produced}"
|
||||
done
|
||||
|
||||
# Globals so the EXIT trap can still see them after this function returns.
|
||||
work=$(mktemp -d)
|
||||
stage=$(mktemp -d)
|
||||
cleanup() {
|
||||
rm -rf "${work:-}" "${stage:-}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
local zip_dir="$work/zip"
|
||||
mkdir -p "$zip_dir"
|
||||
cp "$binary" "$zip_dir/rustfs"
|
||||
chmod 755 "$zip_dir/rustfs"
|
||||
local index=0
|
||||
for extra in "${absolute_extras[@]}"; do
|
||||
cp "$extra" "$zip_dir/${extra_names[$index]}"
|
||||
index=$((index + 1))
|
||||
done
|
||||
(
|
||||
cd "$zip_dir"
|
||||
zip "$stage/$zip_file" rustfs "${extra_names[@]}"
|
||||
)
|
||||
|
||||
local members expected
|
||||
members=$(unzip -Z1 "$stage/$zip_file" | sort)
|
||||
expected=$(printf '%s\n' rustfs "${extra_names[@]}" | sort)
|
||||
[[ "$members" == "$expected" ]] || die "zip members do not match the release archive"
|
||||
[[ "$(unzip -p "$stage/$zip_file" rustfs | sha256sum | awk '{print $1}')" == "$(sha256_file "$binary")" ]] ||
|
||||
die "zip binary differs from the packaged executable"
|
||||
|
||||
ln -s "$repo_root/scripts" "$work/scripts"
|
||||
ln -s "$repo_root/deploy" "$work/deploy"
|
||||
cp "$repo_root/LICENSE" "$repo_root/README.md" "$work/"
|
||||
mkdir -p "$work/bin"
|
||||
cp "$binary" "$work/bin/rustfs"
|
||||
chmod 755 "$work/bin/rustfs"
|
||||
|
||||
local deb_arch deb_depends rpm_depends
|
||||
case "$arch" in
|
||||
x86_64) deb_arch=amd64 ;;
|
||||
aarch64) deb_arch=arm64 ;;
|
||||
*) die "unsupported architecture: ${arch}" ;;
|
||||
esac
|
||||
case "$libc" in
|
||||
gnu)
|
||||
deb_depends='libc6 (>= 2.31)'
|
||||
rpm_depends='glibc >= 2.31'
|
||||
;;
|
||||
musl)
|
||||
deb_depends=''
|
||||
rpm_depends=''
|
||||
;;
|
||||
*) die "unsupported libc: ${libc}" ;;
|
||||
esac
|
||||
|
||||
(
|
||||
cd "$work"
|
||||
GITHUB_OUTPUT="$work/outputs" \
|
||||
DEB_VERSION="$deb_version" \
|
||||
RPM_VERSION="$rpm_version" \
|
||||
RPM_RELEASE="$rpm_release" \
|
||||
DEB_FILE="$deb_file" \
|
||||
RPM_FILE="$rpm_file" \
|
||||
DEB_ARCH="$deb_arch" \
|
||||
RPM_ARCH="$arch" \
|
||||
DEB_DEPENDS="$deb_depends" \
|
||||
RPM_DEPENDS="$rpm_depends" \
|
||||
LIBC="$libc" \
|
||||
bash "$repo_root/scripts/release/build_deb.sh"
|
||||
GITHUB_OUTPUT="$work/outputs" \
|
||||
DEB_VERSION="$deb_version" \
|
||||
RPM_VERSION="$rpm_version" \
|
||||
RPM_RELEASE="$rpm_release" \
|
||||
DEB_FILE="$deb_file" \
|
||||
RPM_FILE="$rpm_file" \
|
||||
DEB_ARCH="$deb_arch" \
|
||||
RPM_ARCH="$arch" \
|
||||
DEB_DEPENDS="$deb_depends" \
|
||||
RPM_DEPENDS="$rpm_depends" \
|
||||
LIBC="$libc" \
|
||||
bash "$repo_root/scripts/release/build_rpm.sh"
|
||||
)
|
||||
cp "$work/$deb_file" "$stage/$deb_file"
|
||||
cp "$work/$rpm_file" "$stage/$rpm_file"
|
||||
|
||||
local unpacked expected_hash actual_hash
|
||||
expected_hash=$(sha256_file "$binary")
|
||||
unpacked="$work/unpacked-deb"
|
||||
mkdir -p "$unpacked"
|
||||
dpkg-deb --extract "$stage/$deb_file" "$unpacked"
|
||||
actual_hash=$(sha256_file "$unpacked/usr/bin/rustfs")
|
||||
[[ "$actual_hash" == "$expected_hash" ]] || die "deb binary differs from the packaged executable"
|
||||
[[ "$(dpkg-deb -f "$stage/$deb_file" Package)" == rustfs ]] || die "deb package name mismatch"
|
||||
[[ "$(dpkg-deb -f "$stage/$deb_file" Version)" == "$deb_version" ]] || die "deb version mismatch"
|
||||
[[ "$(dpkg-deb -f "$stage/$deb_file" Architecture)" == "$deb_arch" ]] || die "deb architecture mismatch"
|
||||
|
||||
unpacked="$work/unpacked-rpm"
|
||||
mkdir -p "$unpacked"
|
||||
rpm2cpio "$stage/$rpm_file" | (cd "$unpacked" && cpio --extract --make-directories --quiet ./usr/bin/rustfs)
|
||||
actual_hash=$(sha256_file "$unpacked/usr/bin/rustfs")
|
||||
[[ "$actual_hash" == "$expected_hash" ]] || die "rpm binary differs from the packaged executable"
|
||||
local rpm_metadata expected_metadata
|
||||
rpm_metadata=$(rpm -qp --qf '%{NAME}\n%{VERSION}\n%{RELEASE}\n%{ARCH}\n' "$stage/$rpm_file")
|
||||
expected_metadata=$(printf 'rustfs\n%s\n%s\n%s' "$rpm_version" "$rpm_release" "$arch")
|
||||
[[ "$rpm_metadata" == "$expected_metadata" ]] || die "rpm metadata mismatch"
|
||||
|
||||
mv "$stage/$zip_file" "$output/$zip_file"
|
||||
mv "$stage/$deb_file" "$output/$deb_file"
|
||||
mv "$stage/$rpm_file" "$output/$rpm_file"
|
||||
emit_resolved
|
||||
}
|
||||
|
||||
main() {
|
||||
local command=${1:-}
|
||||
case "$command" in
|
||||
matrix)
|
||||
[[ $# -eq 2 ]] || usage
|
||||
emit_matrix "$2"
|
||||
;;
|
||||
validate)
|
||||
shift
|
||||
parse_identity "$@"
|
||||
load_versions
|
||||
emit_resolved
|
||||
;;
|
||||
package)
|
||||
shift
|
||||
package_binary "$@"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Contract tests for manual origin/main packaging."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "release" / "package_manual_artifacts.sh"
|
||||
WORKFLOW = ROOT / ".github" / "workflows" / "manual-main-packages.yml"
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from ci_gate import workflow_document # noqa: E402
|
||||
import validation_registry as registry # noqa: E402
|
||||
|
||||
|
||||
def run_script(*args, check=False):
|
||||
return subprocess.run(["bash", str(SCRIPT), *args], cwd=ROOT, text=True,
|
||||
capture_output=True, timeout=60, check=check)
|
||||
|
||||
|
||||
def trigger(text):
|
||||
match = re.search(r"^on:\n(.*?)(?=^[^ #\n]|\Z)", text, re.M | re.S)
|
||||
if match is None:
|
||||
raise AssertionError("workflow has no event declaration")
|
||||
return re.findall(r"^ (\w+):", match.group(1), re.M)
|
||||
|
||||
|
||||
class VersionAndMatrixTests(unittest.TestCase):
|
||||
def test_release_and_prerelease_names_come_from_package_versions(self):
|
||||
cases = {
|
||||
("1.2.3", "x86_64", "gnu"): """\
|
||||
build_type=release
|
||||
deb_version=1.2.3
|
||||
rpm_version=1.2.3
|
||||
rpm_release=1
|
||||
deb_file=rustfs-linux-x86_64-gnu-v1.2.3.deb
|
||||
rpm_file=rustfs-linux-x86_64-gnu-v1.2.3.rpm
|
||||
zip_file=rustfs-linux-x86_64-gnu-v1.2.3.zip
|
||||
""",
|
||||
("1.2.3-alpha.1", "aarch64", "gnu"): """\
|
||||
build_type=prerelease
|
||||
deb_version=1.2.3~alpha.1
|
||||
rpm_version=1.2.3~alpha.1
|
||||
rpm_release=1
|
||||
deb_file=rustfs-linux-aarch64-gnu-v1.2.3-alpha.1.deb
|
||||
rpm_file=rustfs-linux-aarch64-gnu-v1.2.3-alpha.1.rpm
|
||||
zip_file=rustfs-linux-aarch64-gnu-v1.2.3-alpha.1.zip
|
||||
""",
|
||||
("1.2.3-beta.2", "x86_64", "musl"): """\
|
||||
build_type=prerelease
|
||||
deb_version=1.2.3~beta.2
|
||||
rpm_version=1.2.3~beta.2
|
||||
rpm_release=1
|
||||
deb_file=rustfs-linux-x86_64-musl-v1.2.3-beta.2.deb
|
||||
rpm_file=rustfs-linux-x86_64-musl-v1.2.3-beta.2.rpm
|
||||
zip_file=rustfs-linux-x86_64-musl-v1.2.3-beta.2.zip
|
||||
""",
|
||||
("1.2.3-rc.4", "aarch64", "musl"): """\
|
||||
build_type=prerelease
|
||||
deb_version=1.2.3~rc.4
|
||||
rpm_version=1.2.3~rc.4
|
||||
rpm_release=1
|
||||
deb_file=rustfs-linux-aarch64-musl-v1.2.3-rc.4.deb
|
||||
rpm_file=rustfs-linux-aarch64-musl-v1.2.3-rc.4.rpm
|
||||
zip_file=rustfs-linux-aarch64-musl-v1.2.3-rc.4.zip
|
||||
""",
|
||||
}
|
||||
for (version, arch, libc), expected in cases.items():
|
||||
with self.subTest(version=version, arch=arch, libc=libc):
|
||||
result = run_script("validate", "--version", version, "--arch", arch, "--libc", libc)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout, expected)
|
||||
|
||||
def test_invalid_versions_fail_without_stdout(self):
|
||||
for version in ("", "v1.2.3", "1.2", "01.2.3", "1.2.3-preview.1", "1.2.3-alpha",
|
||||
"1.2.3-rc.01", "1.2.3-rc.1-extra", "1.2.3 ", "1.2.3$(id)", "dev-abcdef"):
|
||||
with self.subTest(version=version):
|
||||
result = run_script("validate", "--version", version, "--arch", "x86_64", "--libc", "gnu")
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, "")
|
||||
self.assertIn("invalid package version", result.stderr)
|
||||
self.assertIn("not a release:", result.stderr)
|
||||
self.assertIn("not a prerelease:", result.stderr)
|
||||
|
||||
def test_invalid_version_does_not_create_packages(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
binary = root / "rustfs"
|
||||
binary.write_bytes(b"not packaged")
|
||||
output = root / "dist"
|
||||
result = run_script("package", "--version", "v1.2.3", "--arch", "x86_64", "--libc", "gnu",
|
||||
"--binary", str(binary), "--output", str(output))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, "")
|
||||
self.assertFalse(output.exists())
|
||||
|
||||
def test_catalog_members_must_be_the_official_pair(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
binary = root / "rustfs"
|
||||
binary.write_bytes(b"not packaged")
|
||||
catalog = root / "rustfs.cpu-symbol-catalog.json"
|
||||
catalog.write_text("{}\n", encoding="utf-8")
|
||||
result = run_script("package", "--version", "1.2.3", "--arch", "x86_64", "--libc", "gnu",
|
||||
"--binary", str(binary), "--output", str(root / "dist"),
|
||||
"--zip-extra", str(catalog))
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("packaged together", result.stderr)
|
||||
self.assertFalse((root / "dist").exists())
|
||||
other = root / "notes.txt"
|
||||
other.write_text("nope\n", encoding="utf-8")
|
||||
rejected = run_script("package", "--version", "1.2.3", "--arch", "x86_64", "--libc", "musl",
|
||||
"--binary", str(binary), "--output", str(root / "dist"),
|
||||
"--zip-extra", str(other))
|
||||
self.assertNotEqual(rejected.returncode, 0)
|
||||
self.assertIn("unsupported zip member", rejected.stderr)
|
||||
|
||||
def test_architecture_matrix_matches_official_linux_rows(self):
|
||||
source = (ROOT / ".github" / "workflows" / "build.yml").read_text(encoding="utf-8")
|
||||
official = json.loads(re.search(r"all='(\{.*?\})'", source, re.S).group(1))["include"]
|
||||
linux = [row for row in official if row["platform"] == "linux"]
|
||||
|
||||
def derived(row):
|
||||
arch = "aarch64" if row["target"].startswith("aarch64-") else "x86_64"
|
||||
libc = row["target_id"].rsplit("-", 1)[1]
|
||||
return {**row, "arch": arch, "libc": libc, "build_os": f"linux-{arch}"}
|
||||
|
||||
expected = {
|
||||
"all": [derived(row) for row in linux],
|
||||
"x86": [derived(row) for row in linux if row["target"].startswith("x86_64-")],
|
||||
"arm": [derived(row) for row in linux if row["target"].startswith("aarch64-")],
|
||||
}
|
||||
self.assertEqual([row["target_id"] for row in expected["x86"]],
|
||||
["linux-x86_64-musl", "linux-x86_64-gnu"])
|
||||
self.assertEqual([row["target_id"] for row in expected["arm"]],
|
||||
["linux-aarch64-musl", "linux-aarch64-gnu"])
|
||||
for choice, rows in expected.items():
|
||||
with self.subTest(choice=choice):
|
||||
result = run_script("matrix", choice, check=True)
|
||||
self.assertEqual(json.loads(result.stdout), {"include": rows})
|
||||
for choice in ("", "ALL", "linux", "x86_64", "x86,arm"):
|
||||
with self.subTest(choice=choice):
|
||||
result = run_script("matrix", choice)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, "")
|
||||
self.assertIn("unsupported architecture", result.stderr)
|
||||
|
||||
|
||||
class WorkflowContractTests(unittest.TestCase):
|
||||
def test_dispatch_builds_origin_main_and_only_uploads_artifacts(self):
|
||||
text = WORKFLOW.read_text(encoding="utf-8")
|
||||
self.assertEqual(trigger(text), ["workflow_dispatch"])
|
||||
document = workflow_document(text)
|
||||
events = document.get("on", document.get(True))
|
||||
self.assertEqual(set(events), {"workflow_dispatch"})
|
||||
inputs = events["workflow_dispatch"]["inputs"]
|
||||
self.assertEqual(inputs["architecture"]["type"], "choice")
|
||||
self.assertEqual(inputs["architecture"]["options"], ["x86", "arm", "all"])
|
||||
self.assertTrue(inputs["architecture"]["required"])
|
||||
self.assertEqual(inputs["version"]["type"], "string")
|
||||
self.assertTrue(inputs["version"]["required"])
|
||||
self.assertEqual(document["permissions"], {"contents": "read"})
|
||||
self.assertEqual(re.findall(r"^\s+ref:\s*(.+)$", text, re.M),
|
||||
["main", "${{ needs.prepare.outputs.source_sha }}"])
|
||||
self.assertIn("refs/heads/main", text)
|
||||
self.assertNotIn("github.sha", text)
|
||||
self.assertIn("package_manual_artifacts.sh validate", text)
|
||||
self.assertIn("package_manual_artifacts.sh matrix", text)
|
||||
self.assertIn("package_manual_artifacts.sh package", text)
|
||||
self.assertIn("RUSTFS_BUILD_VERSION: ${{ needs.prepare.outputs.version }}", text)
|
||||
self.assertIn("cargo zigbuild --release --locked", text)
|
||||
self.assertIn("cargo build --release --locked", text)
|
||||
self.assertIn("scripts/release/check_glibc.py", text)
|
||||
self.assertIn('cache-save-if: \'false\'', text)
|
||||
self.assertIn("actions/upload-artifact@", text)
|
||||
self.assertIn("dist/*", text)
|
||||
for forbidden in ("contents: write", "aws s3", "gh release", "softprops/action-gh-release",
|
||||
"docker push", "R2_ACCESS_KEY_ID", "workflow_call:"):
|
||||
self.assertNotIn(forbidden, text)
|
||||
self.assertEqual(text.count("persist-credentials: false"), text.count("actions/checkout@"))
|
||||
item = registry.workflow_entry(".github/workflows/manual-main-packages.yml")
|
||||
self.assertEqual(item["name"], "Manual main packages")
|
||||
self.assertEqual(item["events"], ["workflow_dispatch"])
|
||||
self.assertEqual(item["schedule"], [])
|
||||
self.assertEqual(item["monitor"], "none")
|
||||
self.assertNotEqual(item["origin"], "baseline")
|
||||
|
||||
def test_registry_accepts_the_added_workflow(self):
|
||||
self.assertTrue(registry.check_wiring(registry.load_registry()))
|
||||
|
||||
|
||||
class PackageToolTests(unittest.TestCase):
|
||||
def test_zip_deb_and_rpm_wrap_the_same_binary_when_tools_exist(self):
|
||||
required = ("dpkg-deb", "fakeroot", "rpm", "rpm2cpio", "cpio", "fpm", "zip", "unzip")
|
||||
missing = [tool for tool in required if shutil.which(tool) is None]
|
||||
if missing or os.name != "posix":
|
||||
self.skipTest("packaging tools not installed: " + ", ".join(missing))
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
payload = b"manual main package fixture\n"
|
||||
binary = root / "rustfs"
|
||||
binary.write_bytes(payload)
|
||||
result = run_script("package", "--version", "9.9.9-rc.8", "--arch", "x86_64", "--libc", "gnu",
|
||||
"--binary", str(binary), "--output", str(root / "dist"))
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
dist = root / "dist"
|
||||
self.assertEqual(sorted(path.name for path in dist.iterdir()), [
|
||||
"rustfs-linux-x86_64-gnu-v9.9.9-rc.8.deb",
|
||||
"rustfs-linux-x86_64-gnu-v9.9.9-rc.8.rpm",
|
||||
"rustfs-linux-x86_64-gnu-v9.9.9-rc.8.zip",
|
||||
])
|
||||
self.assertIn("deb_version=9.9.9~rc.8", result.stdout)
|
||||
self.assertIn("zip_file=rustfs-linux-x86_64-gnu-v9.9.9-rc.8.zip", result.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -11,6 +11,7 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent / "release"))
|
||||
@@ -63,6 +64,57 @@ class PackageIntegrationTests(unittest.TestCase):
|
||||
deliver.package(fixtures.inputs(), "linux-x86_64-musl", root / "output")
|
||||
self.assertFalse(list((root / "output").glob("packages-*.json")))
|
||||
|
||||
def test_manual_packages_match_release_names_and_bytes(self):
|
||||
script = Path(__file__).resolve().parent / "release" / "package_manual_artifacts.sh"
|
||||
cases = (
|
||||
("9.9.9-rc.8", "x86_64", "gnu", True),
|
||||
("9.9.9", "aarch64", "musl", False),
|
||||
)
|
||||
for version, arch, libc, with_catalog in cases:
|
||||
with self.subTest(version=version, arch=arch, libc=libc), tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
payload = f"manual-package:{version}:{arch}:{libc}\n".encode()
|
||||
binary = root / "rustfs"
|
||||
binary.write_bytes(payload)
|
||||
command = ["bash", str(script), "package", "--version", version, "--arch", arch, "--libc", libc,
|
||||
"--binary", str(binary), "--output", str(root / "dist")]
|
||||
if with_catalog:
|
||||
for name in ("rustfs.cpu-symbol-catalog.json", "rustfs.cpu-symbol-catalog.sha256"):
|
||||
(root / name).write_text(name + "\n", encoding="utf-8")
|
||||
command.extend(["--zip-extra", str(root / name)])
|
||||
subprocess.run(command, check=True, text=True, timeout=180)
|
||||
stem = f"rustfs-linux-{arch}-{libc}-v{version}"
|
||||
dist = root / "dist"
|
||||
self.assertEqual(sorted(path.name for path in dist.iterdir()),
|
||||
[f"{stem}.deb", f"{stem}.rpm", f"{stem}.zip"])
|
||||
expected_hash = hashlib.sha256(payload).hexdigest()
|
||||
with zipfile.ZipFile(dist / f"{stem}.zip") as archive:
|
||||
names = sorted(archive.namelist())
|
||||
self.assertEqual(hashlib.sha256(archive.read("rustfs")).hexdigest(), expected_hash)
|
||||
if with_catalog:
|
||||
self.assertEqual(names, ["rustfs", "rustfs.cpu-symbol-catalog.json", "rustfs.cpu-symbol-catalog.sha256"])
|
||||
else:
|
||||
self.assertEqual(names, ["rustfs"])
|
||||
deb = dist / f"{stem}.deb"
|
||||
self.assertEqual(subprocess.check_output(["dpkg-deb", "-f", deb, "Version"], text=True).strip(),
|
||||
version.replace("-", "~"))
|
||||
self.assertEqual(subprocess.check_output(["dpkg-deb", "-f", deb, "Architecture"], text=True).strip(),
|
||||
"amd64" if arch == "x86_64" else "arm64")
|
||||
depends = subprocess.check_output(["dpkg-deb", "-f", deb, "Depends"], text=True).strip()
|
||||
self.assertEqual(depends, "libc6 (>= 2.31)" if libc == "gnu" else "")
|
||||
unpacked = root / "deb"
|
||||
subprocess.run(["dpkg-deb", "--extract", str(deb), str(unpacked)], check=True)
|
||||
self.assertEqual(hashlib.sha256((unpacked / "usr/bin/rustfs").read_bytes()).hexdigest(), expected_hash)
|
||||
rpm = dist / f"{stem}.rpm"
|
||||
identity = subprocess.check_output(
|
||||
["rpm", "-qp", "--qf", "%{NAME} %{VERSION} %{RELEASE} %{ARCH}", str(rpm)], text=True)
|
||||
self.assertEqual(identity, f"rustfs {version.replace('-', '~')} 1 {arch}")
|
||||
requires = subprocess.check_output(["rpm", "-qp", "--requires", str(rpm)], text=True)
|
||||
if libc == "gnu":
|
||||
self.assertIn("glibc >= 2.31", requires)
|
||||
else:
|
||||
self.assertNotIn("glibc", requires)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user