mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-16 21:14:45 +00:00
Compare commits
163 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| aca434fdfc | |||
| 4c569451e1 | |||
| f83b964f2c | |||
| 117c9dc197 | |||
| 46f750c6c9 | |||
| 1f8970ba1b | |||
| 0c44fd3bb8 | |||
| 4cb724e64b | |||
| e8ee409f6b | |||
| 08e88f87eb | |||
| 0525b3dc04 | |||
| b8225fb52d | |||
| a1868cf6bb | |||
| a2acae3944 | |||
| 4713e21231 | |||
| 95fc8b7b01 | |||
| 2873de8de9 | |||
| 6cba2bd002 | |||
| 7c8e4c7711 | |||
| e8847f6f1d | |||
| ee7523bab6 | |||
| ca8769d7c2 | |||
| 83f3fbf2cd | |||
| f56477eb53 | |||
| 2f55ddb595 | |||
| 44e147e8eb | |||
| dd760dfa56 | |||
| 7be29ee088 | |||
| 70a0267d01 | |||
| 28ac9de62a | |||
| da6c626f2a | |||
| e2fee7cb2b | |||
| a115748cda | |||
| 1f7f5b8532 | |||
| 0721822cfa | |||
| 0b7fd7a367 | |||
| f3b0806547 | |||
| 1fbcacb2b6 | |||
| 7729e8c337 | |||
| a537b6ea98 | |||
| 3805f2970e | |||
| 62f0385769 | |||
| c312102bab | |||
| 35e6c8077d | |||
| 007c3b204b | |||
| 1c3fe4747e | |||
| 032566507a | |||
| 42ab123d69 | |||
| 22f6c668da | |||
| 9829661a98 | |||
| ee3ef73630 | |||
| 4c7cc8a943 | |||
| 42ee5b151b | |||
| 3cda284b4c | |||
| fb053dfb85 | |||
| 9707aa4f1f | |||
| 197e682488 | |||
| 0e0744cb49 | |||
| 20872bba63 | |||
| 30d8aee534 | |||
| 05e6fd2085 | |||
| 3f861328d8 | |||
| 210e1b214e | |||
| 55d931353f | |||
| 0e1e2e0003 | |||
| b3e60a8882 | |||
| e6a1f542a1 | |||
| 662781e45b | |||
| 7aceffc68c | |||
| e0b53cc8d9 | |||
| db370e3241 | |||
| ffe6bc86d3 | |||
| fc09148f08 | |||
| c9e1464a2a | |||
| 5d6358a0a5 | |||
| 0e3028d29e | |||
| dde2169c83 | |||
| 0b586c37e9 | |||
| 365014a385 | |||
| 7946b094f9 | |||
| 443e8afee1 | |||
| 782e3e5209 | |||
| cfffb1700e | |||
| cd62f3ba4f | |||
| 7bc419e368 | |||
| 7269243de7 | |||
| c4a7423f99 | |||
| 2db4ed244a | |||
| 93a61df1cf | |||
| feec0e9e4a | |||
| 38baeeb11d | |||
| 1af039c972 | |||
| a2fc7c13b4 | |||
| dbe01e5cb1 | |||
| ffc27fa6db | |||
| 7e8edab0d3 | |||
| 22057ad4c5 | |||
| 279a88ab56 | |||
| b4abe64ada | |||
| add363f084 | |||
| 1f2dbe36e5 | |||
| 67fbafa13e | |||
| 1000860cb8 | |||
| b783f404ad | |||
| 11a991c777 | |||
| 3f2b6f690a | |||
| 88b626e7a0 | |||
| 8cf6187ff4 | |||
| a0a0cee9eb | |||
| 80fff4b5c5 | |||
| d91dfcedd2 | |||
| 79c4fedf8c | |||
| 73c846e09c | |||
| 6938c467d0 | |||
| 5323df7049 | |||
| 55b6fbd260 | |||
| dd677cc121 | |||
| 0c2f327974 | |||
| df42cf1ce8 | |||
| d290d46b34 | |||
| 0d297ed8cd | |||
| b8dffa2f1d | |||
| 62b6f5f3fc | |||
| be0986115c | |||
| 1bd91c8dad | |||
| dcb22bd4d5 | |||
| fa17718444 | |||
| 99abffc57a | |||
| fc061c7930 | |||
| d86bf1360f | |||
| f6097d1df2 | |||
| 7e5d32eecf | |||
| 72da0e21c4 | |||
| 78a0cf321c | |||
| cbe38a120f | |||
| db0828aa1c | |||
| 3ca889b099 | |||
| e6112c526f | |||
| c5967686f1 | |||
| 859de565ad | |||
| 1f7cc593e7 | |||
| 1d5a50f094 | |||
| 53245be9f4 | |||
| c47432e353 | |||
| f622c1bce0 | |||
| a99d8f4f89 | |||
| 4d15b81623 | |||
| 836d30192f | |||
| 860c61ee6d | |||
| f600af4704 | |||
| 9e747e8475 | |||
| 96fc067765 | |||
| e809b4899e | |||
| 584f03f300 | |||
| d52d6fade7 | |||
| 3623d6db2d | |||
| 484f1254ca | |||
| c9c16efbb6 | |||
| f2a59073ca | |||
| c1af94d1d1 | |||
| f84bbfe28f | |||
| 378a2ff7f8 | |||
| b88b53aa6a |
@@ -70,6 +70,11 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
||||
@echo "📦 Checking s3s footprint ratchet..."
|
||||
./scripts/check_s3s_footprint.sh
|
||||
|
||||
.PHONY: release-feature-boundary-check
|
||||
release-feature-boundary-check: ## Check no release-reachable feature enables a debug-only one
|
||||
@echo "🚧 Checking release feature boundaries..."
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_connect_release_boundary.py --features-only
|
||||
|
||||
.PHONY: test-count-ratchet-check
|
||||
test-count-ratchet-check: ## Check no workspace crate fell below its test floor or rose above its #[ignore] ceiling in the committed baseline
|
||||
@echo "🧪 Checking per-crate test-count ratchet..."
|
||||
|
||||
@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check test-seams-check test-retirements-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check release-feature-boundary-check test-count-ratchet-check test-seams-check test-retirements-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check test-seams-check test-retirements-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check release-feature-boundary-check test-count-ratchet-check test-seams-check test-retirements-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
|
||||
@@ -107,6 +107,10 @@ runs:
|
||||
git cat-file -e "$FOOTPRINT_BASE^{commit}" 2>/dev/null || git fetch --no-tags --depth=1 origin "$FOOTPRINT_BASE"
|
||||
./scripts/check_s3s_footprint.sh --base "$FOOTPRINT_BASE"
|
||||
|
||||
- name: Check release feature boundaries
|
||||
shell: bash
|
||||
run: python3 scripts/check_connect_release_boundary.py --features-only
|
||||
|
||||
- name: Check per-crate test-count ratchet
|
||||
shell: bash
|
||||
run: ./scripts/check_test_count_ratchet.sh
|
||||
|
||||
@@ -118,7 +118,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=7$' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=8$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'gateway_smoke=true' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -469,7 +469,7 @@ jobs:
|
||||
# hold. `cargo check --release` is metadata-only and needs no
|
||||
# link-concurrency limit; it is cold because ci-dev holds no release
|
||||
# artifacts.
|
||||
- name: Reject short credentials in release builds
|
||||
- name: Reject debug-only features in release builds
|
||||
run: python3 scripts/check_connect_release_boundary.py
|
||||
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
@@ -676,6 +676,54 @@ jobs:
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
test-and-lint-gateway-stack:
|
||||
name: Test and Lint (gateway-stack)
|
||||
# The gateway S3 stack (rustfs/backlog#2734) sits behind the rustfs
|
||||
# `gateway-stack` feature, which no other pull-request lane lints or tests.
|
||||
# Unlike the dormant feature lanes above, this one runs for every code
|
||||
# change: each operation slice puts its handler tests and its refusal-matrix
|
||||
# row behind the feature, and a lane that ran only on demand would let them
|
||||
# rot behind a green pull request.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# Self-hosted for build scale and cache ownership: a workspace-scale Cargo
|
||||
# build that reads the ci-dev cache written on this image.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
# A feature that does not compile fails here; nothing below is skipped
|
||||
# into a pass.
|
||||
- name: Run clippy with gateway-stack
|
||||
run: cargo clippy --locked -p rustfs --all-targets --features gateway-stack -- -D warnings
|
||||
|
||||
# The gateway stack's own tests: the assembly, the operation table and
|
||||
# its completeness, the shared handler sequence, the stack selection, and
|
||||
# the embedded server driven through the whole middleware stack. The
|
||||
# filter names modules, never operations.
|
||||
- name: Run tests with gateway-stack
|
||||
env:
|
||||
# The same bounded link concurrency as the other feature lanes.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
cargo nextest run --profile ci --locked -p rustfs --features gateway-stack --no-tests fail \
|
||||
--status-level all --final-status-level all \
|
||||
-E 'test(/gateway_stack|gateway_handlers|server::stack::/) | binary(embedded_gateway_stack_test)'
|
||||
|
||||
build-rustfs-debug-binary:
|
||||
name: Build RustFS Debug Binary
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
@@ -1564,6 +1612,7 @@ jobs:
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- uring-integration
|
||||
- e2e-tests
|
||||
@@ -1599,6 +1648,7 @@ jobs:
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- build-rustfs-debug-binary-rio-v2
|
||||
- uring-integration
|
||||
|
||||
Generated
+2
@@ -9633,6 +9633,8 @@ dependencies = [
|
||||
"rustfs-ecstore",
|
||||
"rustfs-extension-schema",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-gateway",
|
||||
"rustfs-gateway-adapter",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-heal",
|
||||
"rustfs-heal-contracts",
|
||||
|
||||
@@ -70,15 +70,18 @@ impl<T: Store> IamKeyLookup for IamSys<T> {
|
||||
|
||||
/// A [`CredentialProvider`] over RustFS IAM and the live root credentials.
|
||||
///
|
||||
/// Built only by [`crate::IamAdapters::new`], so it always reads the same [`RootSource`] as the
|
||||
/// policy source it was assembled with.
|
||||
/// A deployment that also installs the policy source builds both through
|
||||
/// [`crate::IamAdapters::new`], so the two read the same [`RootSource`]; one that installs no
|
||||
/// policy source builds this half alone with [`IamCredentials::new`].
|
||||
pub struct IamCredentials {
|
||||
iam: Arc<dyn IamKeyLookup>,
|
||||
root: Arc<dyn RootSource>,
|
||||
}
|
||||
|
||||
impl IamCredentials {
|
||||
pub(crate) fn new(iam: Arc<dyn IamKeyLookup>, root: Arc<dyn RootSource>) -> Self {
|
||||
/// The provider over `iam`, with `root` read on every lookup.
|
||||
#[must_use]
|
||||
pub fn new(iam: Arc<dyn IamKeyLookup>, root: Arc<dyn RootSource>) -> Self {
|
||||
Self { iam, root }
|
||||
}
|
||||
|
||||
|
||||
@@ -20,8 +20,9 @@
|
||||
//! NOT responsible for: evaluating any policy, choosing which policy documents apply (that stays
|
||||
//! in `IamSys::is_allowed`, which owns the STS, OIDC, session-policy and service-account rules), or
|
||||
//! verifying the signature or the session token (the gateway did both before calling this).
|
||||
//! Upstream: the gateway's verified [`Identity`]. Downstream: the RustFS authorizer, which reads
|
||||
//! the payload back with `PolicySnapshot::get::<RustfsPrincipal>()`.
|
||||
//! Upstream: the gateway's verified [`Identity`]. Downstream: an authorizer that reads the payload
|
||||
//! back with `PolicySnapshot::get::<RustfsPrincipal>()`. The RustFS gateway assembly installs no
|
||||
//! policy source: its handlers resolve the caller through the legacy access phase.
|
||||
//!
|
||||
//! # Failure classes
|
||||
//!
|
||||
|
||||
@@ -56,7 +56,8 @@ The four site-replication state-writer concurrency proofs reserve the available
|
||||
| PR, non-doc change | `Typos` | `ci.yml` `typos` | Via aggregate | `typos` |
|
||||
| Full selection | `Workspace tests` (serial ILM step) | `ci.yml` `workspace-tests` | Via aggregate | exact `Run ignored ILM integration tests serially` command in the job |
|
||||
| Nightly schedule, manual dispatch | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)`, `Test and Lint (http3)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo nextest run` with the job's `--features` |
|
||||
| Nightly schedule, manual dispatch, full selection | `Connect Feature Boundaries` | `ci.yml` `connect-feature-boundaries` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail; `scripts/check_offline_enrollment_e2e.sh` |
|
||||
| Full selection, every event | `Test and Lint (gateway-stack)` | `ci.yml` `test-and-lint-gateway-stack` | Via aggregate | `cargo clippy -p rustfs --all-targets --features gateway-stack -- -D warnings`; `cargo nextest run -p rustfs --features gateway-stack` with the job's filter |
|
||||
| Nightly schedule, manual dispatch, full selection | `Connect Feature Boundaries` | `ci.yml` `connect-feature-boundaries` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `scripts/check_connect_release_boundary.py`: `cargo check -p rustfs --release` with `connect-e2e-short-credentials` and with `gateway-stack` must each fail at its guard; `scripts/check_offline_enrollment_e2e.sh` |
|
||||
| PR, non-doc change | `Build RustFS Debug Binary` | `ci.yml` `build-rustfs-debug-binary` | Via aggregate; prerequisite for black-box jobs | `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks` (binary plus its `rustfs.e2e.json` sidecar) |
|
||||
| Nightly schedule, manual dispatch, full selection | `io_uring Integration (real)` | `ci.yml` `uring-integration` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
|
||||
| PR, non-doc change | `End-to-End Tests` | `ci.yml` `e2e-tests` | Via aggregate | `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test`, then `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` under the same wrapper; membership guards `scripts/check_test_wiring.py --check-profile e2e-smoke <listing.json>` and `scripts/check_security_smoke_count.sh check <listing.json>` |
|
||||
|
||||
+17
-1
@@ -56,6 +56,13 @@ required-features = ["swift"]
|
||||
name = "swift_object_integration_test"
|
||||
required-features = ["swift"]
|
||||
|
||||
# Starts an embedded server on the gateway stack, which only a build with the
|
||||
# feature can select.
|
||||
[[test]]
|
||||
name = "embedded_gateway_stack_test"
|
||||
path = "tests/embedded_gateway_stack_test/main.rs"
|
||||
required-features = ["gateway-stack"]
|
||||
|
||||
[features]
|
||||
internode-transport = ["rustfs-ecstore/internode-transport", "rustfs-rio/internode-transport"]
|
||||
default = ["ftps", "webdav", "gcs"]
|
||||
@@ -70,12 +77,19 @@ license = []
|
||||
io-scheduler-debug = [] # Enable debug information in I/O scheduler
|
||||
tracing-chunk-debug = [] # Enable per-chunk tracing in data plane (high noise, for debugging only)
|
||||
full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope", "gcs"]
|
||||
e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hooks"]
|
||||
# The debug-only E2E binary also carries the gateway S3 stack, so the gateway smoke
|
||||
# lane runs the same artifact as the legacy one with RUSTFS_S3_STACK as its only change.
|
||||
e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hooks", "gateway-stack"]
|
||||
# Shortens Connect credentials only in debug E2E builds.
|
||||
connect-e2e-short-credentials = []
|
||||
# Builds the dedicated rustfs-cli-e2e target with a build-time public enrollment root.
|
||||
offline-enrollment-e2e-root = []
|
||||
rio-v2 = ["rustfs-ecstore/rio-v2"]
|
||||
# The rustfs-gateway S3 stack, selected at startup with RUSTFS_S3_STACK=gateway
|
||||
# (rustfs/backlog#2734). Under construction: off by default and never enabled in a
|
||||
# release build; the stack refuses to compile without debug assertions, and
|
||||
# `scripts/check_connect_release_boundary.py` keeps every release-reachable feature from enabling it.
|
||||
gateway-stack = ["dep:rustfs-gateway", "dep:rustfs-gateway-adapter"]
|
||||
pyroscope = ["rustfs-obs/pyroscope", "dep:pyroscope", "dep:object", "dep:symbolic-demangle"]
|
||||
# Tokio runtime telemetry. Requires `--cfg tokio_unstable`; use `make build-profiling`.
|
||||
dial9 = ["rustfs-obs/dial9"]
|
||||
@@ -256,6 +270,8 @@ rustfs-notify = { workspace = true }
|
||||
rustfs-obs = { workspace = true }
|
||||
rustfs-policy = { workspace = true }
|
||||
rustfs-protocols = { workspace = true }
|
||||
rustfs-gateway = { workspace = true, optional = true }
|
||||
rustfs-gateway-adapter = { workspace = true, optional = true }
|
||||
rustfs-gateway-types = { workspace = true, features = ["compat-s3s-rustfs"] }
|
||||
rustfs-protos = { workspace = true }
|
||||
rustfs-rio = { workspace = true }
|
||||
|
||||
+12
-18
@@ -26,8 +26,7 @@ use crate::admin::utils::{parse_query_pairs, query_value_exact, query_values_exa
|
||||
use crate::auth::{check_key_valid, constant_time_eq, get_session_token, reject_unsigned_amz_headers_on_sigv4_request};
|
||||
use crate::license::license_check;
|
||||
use crate::server::{
|
||||
ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, is_admin_path, is_sts_query_request,
|
||||
is_table_catalog_path,
|
||||
ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, is_control_plane_route, is_sts_query_request, is_table_catalog_path,
|
||||
};
|
||||
use http::HeaderValue;
|
||||
use http::header::HeaderName;
|
||||
@@ -912,6 +911,15 @@ pub struct S3Router<T> {
|
||||
server_ctx: Option<Arc<ServerContextSlot>>,
|
||||
}
|
||||
|
||||
/// Whether this router answers the request itself rather than an S3 operation: the RustFS extension
|
||||
/// routes and every control-plane route.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) fn claims_route(method: &Method, uri: &Uri, headers: &HeaderMap) -> bool {
|
||||
s3_extension::route_key(method, uri).is_some()
|
||||
|| parse_object_lambda_request(method, uri).is_some()
|
||||
|| is_control_plane_route(method, uri, headers)
|
||||
}
|
||||
|
||||
fn is_public_health_path(path: &str) -> bool {
|
||||
path == HEALTH_PREFIX || path == HEALTH_READY_PATH
|
||||
}
|
||||
@@ -1027,22 +1035,7 @@ where
|
||||
return true;
|
||||
}
|
||||
|
||||
// Profiling endpoints
|
||||
if method == Method::GET && (path == PROFILE_CPU_PATH || path == PROFILE_MEMORY_PATH) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Health check
|
||||
if (method == Method::HEAD || method == Method::GET) && is_public_health_path(path) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// AssumeRole
|
||||
if is_sts_query_request(method, uri, headers) {
|
||||
return true;
|
||||
}
|
||||
|
||||
is_admin_path(path) || is_console_path(path)
|
||||
is_control_plane_route(method, uri, headers)
|
||||
}
|
||||
|
||||
// check_access before call
|
||||
@@ -1247,6 +1240,7 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::admin::operation::{AdminRequest, AdminResponse, Operation as OwnedOperation};
|
||||
use crate::auth::constant_time_eq;
|
||||
use crate::server::{PROFILE_CPU_PATH, is_admin_path};
|
||||
use bytes::Bytes;
|
||||
use http::HeaderMap;
|
||||
use http::Method;
|
||||
|
||||
@@ -58,9 +58,9 @@ use super::storage_api::bucket_usecase::error::StorageError;
|
||||
use super::storage_api::bucket_usecase::helper::{OperationHelper, spawn_background_with_context};
|
||||
use super::storage_api::bucket_usecase::object_utils::to_s3s_xml_etag;
|
||||
use super::storage_api::bucket_usecase::s3_api::bucket::{
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
|
||||
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
|
||||
parse_list_object_versions_params, parse_list_objects_v2_params, rustfs_owner,
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_object_versions_output, build_list_objects_output,
|
||||
build_list_objects_v2_output, encode_list_objects_v2_continuation_token, parse_list_object_versions_params,
|
||||
parse_list_objects_v2_params, rustfs_owner,
|
||||
};
|
||||
use super::storage_api::bucket_usecase::{
|
||||
get_validated_store, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
|
||||
@@ -68,7 +68,9 @@ use super::storage_api::bucket_usecase::{
|
||||
};
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
use super::storage_api::diagnostic;
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::bucket_list_through as list_through;
|
||||
use crate::app::dto_values;
|
||||
use crate::app::object::shared::mark_on_demand_migration_list_local_only;
|
||||
use crate::app::object_data_cache::invalidate_object_data_cache_bucket_after_delete;
|
||||
use crate::app::runtime_sources::{
|
||||
@@ -89,6 +91,7 @@ use http::StatusCode;
|
||||
use metrics::counter;
|
||||
use rustfs_config::RUSTFS_REGION;
|
||||
use rustfs_gateway_types::cors_tagging::{PersistedCorsConfiguration, PersistedTagging};
|
||||
use rustfs_gateway_types::dto as gateway_dto;
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedBucketEncryptionConfiguration, PersistedNotificationConfiguration,
|
||||
PersistedObjectLockConfiguration as ObjectLockConfiguration, PersistedPublicAccessBlockConfiguration,
|
||||
@@ -109,9 +112,8 @@ use rustfs_targets::{
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use rustfs_utils::http::{force_delete_header, get_header};
|
||||
use rustfs_utils::obj::extract_user_defined_metadata;
|
||||
use s3s::dto::{ListBucketsInput, ListBucketsOutput};
|
||||
use s3s::region::Region;
|
||||
use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Result};
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
collections::{HashMap, HashSet},
|
||||
@@ -408,21 +410,40 @@ impl DefaultBucketUsecase {
|
||||
current_object_store_handle_for_context(self.context.as_deref())
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_list_buckets(&self, req: S3Request<ListBucketsInput>) -> S3Result<S3Response<ListBucketsOutput>> {
|
||||
/// Lists the buckets the caller of `envelope` may see, for either edge.
|
||||
///
|
||||
/// The access phase must have installed the caller's `ReqInfo`. A caller allowed
|
||||
/// `s3:ListAllMyBuckets` sees every bucket; any other caller sees the buckets it may list or
|
||||
/// locate. `_input` is not read: RustFS answers one unpaginated listing and ignores `prefix`,
|
||||
/// `max-buckets`, `continuation-token` and `bucket-region`.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// `AccessDenied` for an anonymous caller and for one that may see no bucket; `InternalError`
|
||||
/// when the store or IAM is not initialized or the access phase did not run; otherwise the
|
||||
/// storage error of the listing.
|
||||
#[instrument(level = "debug", skip(self, envelope, _input))]
|
||||
pub async fn execute_list_buckets(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
_input: gateway_dto::ListBucketsInput,
|
||||
) -> rustfs_s3_types::S3Result<gateway_dto::ListBucketsOutput> {
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
if req.credentials.as_ref().is_none_or(|cred| cred.access_key.is_empty()) {
|
||||
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
|
||||
if envelope
|
||||
.credentials()
|
||||
.is_none_or(|credentials| credentials.access_key().is_empty())
|
||||
{
|
||||
return Err(rustfs_s3_types::s3_error!(AccessDenied, "Access Denied"));
|
||||
}
|
||||
|
||||
let iam_authorization = prepare_list_buckets_iam_authorization(&req).await?;
|
||||
let iam_authorization = prepare_list_buckets_iam_authorization(&envelope).await?;
|
||||
let bucket_infos = if iam_authorization.is_allowed("", S3Action::ListAllMyBucketsAction).await {
|
||||
store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?
|
||||
} else {
|
||||
log_list_buckets_iam_implicit_deny(&req)?;
|
||||
log_list_buckets_iam_implicit_deny(&envelope)?;
|
||||
let bucket_infos = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?;
|
||||
let mut visible_bucket_infos = Vec::new();
|
||||
for info in bucket_infos {
|
||||
@@ -441,7 +462,22 @@ impl DefaultBucketUsecase {
|
||||
visible_bucket_infos
|
||||
};
|
||||
|
||||
Ok(S3Response::new(build_list_buckets_output(&bucket_infos)))
|
||||
let buckets = bucket_infos
|
||||
.iter()
|
||||
.map(|info| {
|
||||
Ok(gateway_dto::Bucket {
|
||||
name: dto_values::stored_bucket_name(&info.name)?,
|
||||
creation_date: dto_values::timestamp(info.created.unwrap_or(time::OffsetDateTime::UNIX_EPOCH)),
|
||||
..Default::default()
|
||||
})
|
||||
})
|
||||
.collect::<rustfs_s3_types::S3Result<_>>()?;
|
||||
Ok(gateway_dto::ListBucketsOutput {
|
||||
buckets,
|
||||
owner: dto_values::owner(),
|
||||
continuation_token: None,
|
||||
prefix: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -943,12 +979,13 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_list_buckets_returns_internal_error_when_store_uninitialized() {
|
||||
let input = ListBucketsInput::builder().build().unwrap();
|
||||
|
||||
let req = build_request(input, Method::GET);
|
||||
let req = build_request((), Method::GET);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
|
||||
let err = usecase.execute_list_buckets(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
let err = usecase
|
||||
.execute_list_buckets(RequestEnvelope::from_s3s(&req), gateway_dto::ListBucketsInput::default())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
BucketLocationConstraint, CreateBucketInput, CreateBucketOutput, DeleteBucketInput, DeleteBucketOutput,
|
||||
GetBucketLocationInput, GetBucketLocationOutput, HeadBucketInput, HeadBucketOutput,
|
||||
GetBucketLocationInput, GetBucketLocationOutput,
|
||||
};
|
||||
use crate::app::storage_api::s3::region::Region;
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result};
|
||||
@@ -279,21 +279,27 @@ impl DefaultBucketUsecase {
|
||||
result
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
pub async fn execute_head_bucket(&self, req: S3Request<HeadBucketInput>) -> S3Result<S3Response<HeadBucketOutput>> {
|
||||
record_s3_op(S3Operation::HeadBucket);
|
||||
let input = req.input;
|
||||
|
||||
/// Answers whether `input.bucket` exists, for either edge. The access phase has authorized it.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// `InternalError` when the store is not initialized; otherwise the storage error of the
|
||||
/// lookup, `NoSuchBucket` among them.
|
||||
#[instrument(level = "debug", skip(self, input))]
|
||||
pub async fn execute_head_bucket(
|
||||
&self,
|
||||
input: gateway_dto::HeadBucketInput,
|
||||
) -> rustfs_s3_types::S3Result<gateway_dto::HeadBucketOutput> {
|
||||
let Some(store) = self.object_store() else {
|
||||
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
|
||||
return Err(rustfs_s3_types::s3_error!(InternalError, "Not init"));
|
||||
};
|
||||
|
||||
store
|
||||
.get_bucket_info(&input.bucket, &BucketOptions::default())
|
||||
.get_bucket_info(input.bucket.as_str(), &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
Ok(S3Response::new(HeadBucketOutput::default()))
|
||||
Ok(gateway_dto::HeadBucketOutput::default())
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
@@ -496,15 +502,17 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_head_bucket_returns_internal_error_when_store_uninitialized() {
|
||||
let input = HeadBucketInput::builder().bucket("test-bucket".to_string()).build().unwrap();
|
||||
|
||||
let req = build_request(input, Method::HEAD);
|
||||
let input = gateway_dto::HeadBucketInput {
|
||||
bucket: rustfs_gateway_types::BucketName::new("test-bucket").expect("valid bucket name"),
|
||||
..Default::default()
|
||||
};
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
let before = s3_op_total(S3Operation::HeadBucket);
|
||||
|
||||
let err = usecase.execute_head_bucket(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(s3_op_total(S3Operation::HeadBucket), before + 1);
|
||||
let err = usecase.execute_head_bucket(input).await.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
// Each edge counts the operation once, before it calls the use case.
|
||||
assert_eq!(s3_op_total(S3Operation::HeadBucket), before);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -459,6 +459,9 @@ impl AuthorizationRequest for RequestEnvelope {
|
||||
fn headers(&self) -> &HeaderMap {
|
||||
self.headers()
|
||||
}
|
||||
fn headers_mut(&mut self) -> &mut HeaderMap {
|
||||
&mut self.headers
|
||||
}
|
||||
fn uri(&self) -> &http::Uri {
|
||||
self.uri()
|
||||
}
|
||||
@@ -474,6 +477,9 @@ impl AuthorizationRequest for RequestEnvelope {
|
||||
fn req_info(&self) -> Option<&ReqInfo> {
|
||||
self.req_info()
|
||||
}
|
||||
fn req_info_mut(&mut self) -> Option<&mut ReqInfo> {
|
||||
self.req_info_mut()
|
||||
}
|
||||
fn internal_object_authorization(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Stored values as the gateway's answer types hold them, for every use case that answers in them.
|
||||
//!
|
||||
//! Responsible for: the bucket name, timestamp and owner conversions a use case needs to build a
|
||||
//! gateway output from what the store returned, under the rule in [`crate::app`].
|
||||
//! NOT responsible for: validating names a client sends (the edge that accepted the request did),
|
||||
//! or deciding what a use case lists.
|
||||
//! Upstream: the use cases in `crate::app`. Downstream: `rustfs_gateway_types`.
|
||||
|
||||
use super::storage_api::bucket_usecase::s3_api::bucket::{RUSTFS_OWNER_DISPLAY_NAME, RUSTFS_OWNER_ID};
|
||||
use rustfs_gateway_types::dto::Owner;
|
||||
use rustfs_gateway_types::{BucketName, NamePolicy, NameValidator, Stricter, Timestamp};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
use std::sync::{Arc, LazyLock};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// Adds no rule above the gateway's floor: every name the store holds was admitted by RustFS's own
|
||||
/// create rule when it was made, and the store's listing drops anything else.
|
||||
struct StoredNames;
|
||||
|
||||
impl NameValidator for StoredNames {
|
||||
fn check_bucket(&self, _name: &str) -> Stricter {
|
||||
Stricter::NoOpinion
|
||||
}
|
||||
|
||||
fn check_key(&self, _key: &str) -> Stricter {
|
||||
Stricter::NoOpinion
|
||||
}
|
||||
}
|
||||
|
||||
static STORED_NAMES: LazyLock<NamePolicy> = LazyLock::new(|| NamePolicy::default().with_validator(Arc::new(StoredNames)));
|
||||
|
||||
/// The bucket name the store returned, as an answer carries it.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// `InternalError` for a name the gateway floor refuses. The store's listing never returns one
|
||||
/// (pinned by `dto_values::tests`), so this is a store defect, never a name to leave out.
|
||||
pub(crate) fn stored_bucket_name(name: &str) -> S3Result<BucketName> {
|
||||
BucketName::materialize(name, &STORED_NAMES).map_err(|rejection| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("a stored bucket name the gateway cannot represent: {rejection}"),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// `at` with its sub-second part.
|
||||
pub(crate) fn timestamp(at: OffsetDateTime) -> Timestamp {
|
||||
let seconds = at.unix_timestamp();
|
||||
// `nanosecond()` is below one second by type, the only input `from_secs_nanos` refuses.
|
||||
Timestamp::from_secs_nanos(seconds, at.nanosecond()).expect("a sub-second remainder is always representable")
|
||||
}
|
||||
|
||||
/// The one owner RustFS answers with.
|
||||
pub(crate) fn owner() -> Owner {
|
||||
Owner {
|
||||
display_name: Some(RUSTFS_OWNER_DISPLAY_NAME.to_owned()),
|
||||
id: Some(RUSTFS_OWNER_ID.to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::storage::storage_api::ecstore_capacity::is_reserved_or_invalid_bucket;
|
||||
|
||||
/// Names RustFS's create rule admits and the AWS bucket grammar refuses.
|
||||
const RUSTFS_ONLY_NAMES: [&str; 5] = ["a-s3alias", "sthree-x", "x--ol-s3", "x--x-s3", "xn--abc"];
|
||||
|
||||
#[test]
|
||||
fn names_rustfs_creates_and_aws_refuses_are_represented_unchanged() {
|
||||
for name in RUSTFS_ONLY_NAMES.into_iter().chain(["sftp_host_keys", "UPPER"]) {
|
||||
assert!(BucketName::new(name).is_err(), "{name} must be outside the AWS grammar for this test");
|
||||
let materialized = stored_bucket_name(name).unwrap_or_else(|error| panic!("{name}: {error:?}"));
|
||||
assert_eq!(materialized.as_str(), name);
|
||||
}
|
||||
// Dotted quads the standard address parser refuses. RustFS's create rule refuses every
|
||||
// dotted quad, so none is stored, but the floor would not drop one either.
|
||||
for name in ["256.1.1.1", "01.2.3.4"] {
|
||||
assert_eq!(stored_bucket_name(name).map(|name| name.as_str().to_owned()).ok().as_deref(), Some(name));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_name_the_floor_refuses_is_a_server_fault_not_a_gap_in_the_listing() {
|
||||
for name in ["ab", "a b", "a%2fb", "a/b"] {
|
||||
let error = stored_bucket_name(name).expect_err(name);
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError, "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The store's listing is the only source of stored names a use case answers with, and it
|
||||
/// drops every name its own rule refuses. Every byte in every position the rule constrains,
|
||||
/// and both length bounds: whatever it keeps, the floor represents.
|
||||
#[test]
|
||||
fn every_name_the_store_lists_is_representable() {
|
||||
let mut names: Vec<String> = (0..=u8::MAX)
|
||||
.filter_map(|byte| String::from_utf8(vec![b'a', byte, b'b']).ok())
|
||||
.chain((0..=u8::MAX).filter_map(|byte| String::from_utf8(vec![byte, b'a', b'b']).ok()))
|
||||
.collect();
|
||||
names.extend(["ab", "abc", &"a".repeat(63), &"a".repeat(64)].map(str::to_owned));
|
||||
|
||||
let listed: Vec<&String> = names
|
||||
.iter()
|
||||
.filter(|name| !is_reserved_or_invalid_bucket(name, false))
|
||||
.collect();
|
||||
|
||||
assert!(listed.len() > 60, "the probe must exercise names the store keeps: {}", listed.len());
|
||||
for name in listed {
|
||||
assert!(stored_bucket_name(name).is_ok(), "the store lists {name:?} and the floor refuses it");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_timestamp_keeps_its_sub_second_part() {
|
||||
let at = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_000_123_000_000).expect("valid timestamp");
|
||||
|
||||
let converted = timestamp(at);
|
||||
|
||||
assert_eq!((converted.secs(), converted.subsec_nanos()), (1_700_000_000, 123_000_000));
|
||||
}
|
||||
}
|
||||
@@ -14,11 +14,17 @@
|
||||
|
||||
//! Application layer module entry.
|
||||
//! Concrete use-case modules will be introduced incrementally in Phase 3.
|
||||
//!
|
||||
//! A use case that answers in the gateway's types materialises every stored name under the
|
||||
//! gateway's floor only: the strict wire grammar applies to names a client sends, never to names
|
||||
//! the store returns. A stored value is never dropped or rewritten to fit a stricter scalar
|
||||
//! ([`dto_values`]).
|
||||
|
||||
pub mod admin_usecase;
|
||||
pub(crate) mod bucket_list_through;
|
||||
pub mod bucket_usecase;
|
||||
pub mod context;
|
||||
pub(crate) mod dto_values;
|
||||
pub(crate) mod metadata_route;
|
||||
pub mod multipart_usecase;
|
||||
pub mod object;
|
||||
|
||||
@@ -1690,11 +1690,12 @@ pub(crate) mod timeout_wrapper {
|
||||
pub(crate) mod s3_api {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use crate::storage::storage_api::s3_api_consumer::bucket::{
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
|
||||
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
|
||||
parse_list_object_versions_params, parse_list_objects_v2_params,
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_object_versions_output, build_list_objects_output,
|
||||
build_list_objects_v2_output, encode_list_objects_v2_continuation_token, parse_list_object_versions_params,
|
||||
parse_list_objects_v2_params,
|
||||
};
|
||||
pub(crate) use crate::storage::storage_api::s3_api_consumer::common::rustfs_owner;
|
||||
pub(crate) use crate::storage::storage_api::s3_api_consumer::common::{RUSTFS_OWNER_DISPLAY_NAME, RUSTFS_OWNER_ID};
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
|
||||
@@ -80,7 +80,7 @@ impl FromStr for S3Stack {
|
||||
impl S3Stack {
|
||||
/// Unset selects the default; a set but invalid value, including an empty
|
||||
/// one, is a startup error.
|
||||
fn from_env() -> std::io::Result<Self> {
|
||||
pub(crate) fn from_env() -> std::io::Result<Self> {
|
||||
match std::env::var_os(ENV_RUSTFS_S3_STACK) {
|
||||
None => Ok(Self::default()),
|
||||
Some(value) => value
|
||||
|
||||
@@ -421,6 +421,13 @@ impl Drop for RustFSServer {
|
||||
}
|
||||
}
|
||||
|
||||
/// The S3 operations a server on the gateway stack serves. It answers every other operation
|
||||
/// `501 NotImplemented`.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub fn gateway_operations() -> Vec<&'static str> {
|
||||
crate::server::gateway_operations()
|
||||
}
|
||||
|
||||
/// Find an available TCP port on localhost.
|
||||
///
|
||||
/// Binds to port `0`, reads the OS-assigned port, then releases the socket.
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The shared access phase of the gateway edge: what the legacy `S3Access::check` runs for every
|
||||
//! operation, fed from the gateway's verdict.
|
||||
//!
|
||||
//! Responsible for: [`admit`], which builds the envelope with the access markers, installs the
|
||||
//! caller and runs the operation-aware admission, each through the function the legacy check
|
||||
//! calls (`storage/access.rs`).
|
||||
//! NOT responsible for: an operation's own authorization (`Served::authorize`), or verifying the
|
||||
//! signature (the gateway did, and only a verified caller has a principal).
|
||||
//! Upstream: `GatewayBackend::serve`. Downstream: the storage layer's access functions.
|
||||
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::storage_api::server::http::access::{VerifiedSignature, admit_operation, resolve_req_info};
|
||||
use rustfs_gateway::sig::{SigFamily, SigLocation};
|
||||
use rustfs_gateway::{RequestContextView, RequestPrincipal};
|
||||
use rustfs_s3_types::S3Result;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Runs the shared access phase for one accepted request and returns its envelope.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// What the legacy access check answers: the credential refusals of the key lookup,
|
||||
/// `InvalidRequest` for a misplaced capability query, `AccessDenied` for a refused license, and
|
||||
/// `InternalError` when IAM or the license cannot be read or the request cannot be read back.
|
||||
pub(super) async fn admit(server_ctx: &Arc<ServerContextSlot>, context: &RequestContextView) -> S3Result<RequestEnvelope> {
|
||||
let signature = verified_signature(context);
|
||||
let mut envelope = RequestEnvelope::from_gateway(context, server_ctx, signature)?;
|
||||
let credentials = envelope.credentials();
|
||||
let access_key = credentials.map(|credentials| credentials.access_key().to_owned());
|
||||
let session_token = credentials
|
||||
.and_then(|credentials| credentials.session_token())
|
||||
.unwrap_or_default()
|
||||
.to_owned();
|
||||
let req_info =
|
||||
resolve_req_info(server_ctx, access_key.as_deref(), &session_token, envelope.request_context().cloned()).await?;
|
||||
envelope.install_req_info(req_info);
|
||||
admit_operation(envelope.headers(), envelope.uri().query(), context.operation(), signature)?;
|
||||
Ok(envelope)
|
||||
}
|
||||
|
||||
/// The gateway's verdict as the access markers read it: a SigV4 signature in the header or the
|
||||
/// query. SigV2, a POST policy and an anonymous request mark nothing, as on the legacy edge.
|
||||
pub(super) fn verified_signature(context: &RequestContextView) -> VerifiedSignature {
|
||||
match context.principal().map(RequestPrincipal::scheme) {
|
||||
Some(scheme) if scheme.family() == SigFamily::V4 => VerifiedSignature {
|
||||
sigv4: matches!(scheme.location(), SigLocation::Header | SigLocation::Query),
|
||||
presigned: scheme.location() == SigLocation::Query,
|
||||
},
|
||||
_ => VerifiedSignature::default(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch A2 (rustfs/backlog#2765): bucket policy, policy status, versioning, notification and ACL.
|
||||
//! None is served yet; the batch appends its rows here.
|
||||
|
||||
pub(super) const ROWS: &[super::OpRow] = &[];
|
||||
@@ -0,0 +1,18 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch A1 (rustfs/backlog#2765): bucket CORS, tagging, public access block and encryption.
|
||||
//! None is served yet; the batch appends its rows here.
|
||||
|
||||
pub(super) const ROWS: &[super::OpRow] = &[];
|
||||
@@ -0,0 +1,61 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch A4 (rustfs/backlog#2765): bucket create, delete,
|
||||
//! head and location, website, accelerate, logging and request payment; and ListBuckets.
|
||||
|
||||
use super::{OpRow, Served, row};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::runtime_sources::AppContext;
|
||||
use crate::storage_api::server::http::access::authorize_bucket_request;
|
||||
use rustfs_gateway::{Req, Resp, dto};
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_s3_types::S3Result;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(super) const ROWS: &[OpRow] = &[row::<dto::ListBuckets>(), row::<dto::HeadBucket>()];
|
||||
|
||||
impl Served for dto::ListBuckets {
|
||||
const METRIC: S3Operation = S3Operation::ListBuckets;
|
||||
|
||||
/// The legacy hook checks nothing: the use case decides per caller which buckets it sees.
|
||||
async fn authorize(_envelope: &mut RequestEnvelope, _request: &Req<Self>) -> S3Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn execute(context: Option<Arc<AppContext>>, envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
|
||||
let output = DefaultBucketUsecase::with_context(context)
|
||||
.execute_list_buckets(envelope, request.into_input())
|
||||
.await?;
|
||||
Ok(Resp::new(output))
|
||||
}
|
||||
}
|
||||
|
||||
impl Served for dto::HeadBucket {
|
||||
const METRIC: S3Operation = S3Operation::HeadBucket;
|
||||
|
||||
async fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> S3Result<()> {
|
||||
let action = Action::S3Action(S3Action::ListBucketAction);
|
||||
Ok(authorize_bucket_request(envelope, request.input().bucket.as_str(), action).await?)
|
||||
}
|
||||
|
||||
async fn execute(context: Option<Arc<AppContext>>, _envelope: RequestEnvelope, request: Req<Self>) -> S3Result<Resp<Self>> {
|
||||
let output = DefaultBucketUsecase::with_context(context)
|
||||
.execute_head_bucket(request.into_input())
|
||||
.await?;
|
||||
Ok(Resp::new(output))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch A3 (rustfs/backlog#2765): bucket lifecycle, replication and object lock configuration.
|
||||
//! None is served yet; the batch appends its rows here.
|
||||
|
||||
pub(super) const ROWS: &[super::OpRow] = &[];
|
||||
@@ -0,0 +1,390 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! Builds a [`RequestEnvelope`] from a request the gateway accepted, authenticated and routed.
|
||||
//!
|
||||
//! Responsible for: `RequestEnvelope::from_gateway`, the gateway edge's counterpart of the
|
||||
//! legacy edge's `from_s3s` (`storage/s3_compat/envelope.rs`).
|
||||
//! NOT responsible for: the caller's `ReqInfo` and what an operation's authorization retains;
|
||||
//! `super::access` and `Served::authorize` install those.
|
||||
//! Upstream: `super::access::admit`. Downstream: every use case the gateway stack calls.
|
||||
//!
|
||||
//! Every fact is copied from where the gateway or a RustFS layer in front of it put it: the
|
||||
//! request line and the accepted header lines from the context, the caller from the verified
|
||||
//! principal, and the payloads the connection and request layers attached from the transport
|
||||
//! values the gateway carried through unchanged.
|
||||
|
||||
use crate::app::object::request_body::BodyReadControl;
|
||||
use crate::app::{RequestCredentials, RequestEnvelope, RequestHead};
|
||||
use crate::auth::get_session_token;
|
||||
use crate::runtime_sources::ServerContextSlot;
|
||||
use crate::shared_types::{NativeTlsTransport, RemoteAddr};
|
||||
use crate::storage_api::server::http::access::{AccessMarkers, VerifiedSignature};
|
||||
use crate::storage_api::server::http::request_context::RequestContext;
|
||||
use http::{Extensions, HeaderMap, HeaderValue, Uri};
|
||||
use rustfs_gateway::{RequestContextView, TransportExtensions};
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::sync::Arc;
|
||||
|
||||
impl RequestEnvelope {
|
||||
/// Copies every request fact of `context` the app layer reads, for the server `server_ctx`, with
|
||||
/// the access markers of `signature` and of the routed operation.
|
||||
///
|
||||
/// A payload the request does not carry leaves its field `None`. The caller's secret key is
|
||||
/// copied only when the gateway handed it to this operation. A request that sent no `Host` line,
|
||||
/// which only HTTP/2 and HTTP/3 may, gets one from its authority, as the readers of `Host` expect.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// `InternalError` when the accepted request target cannot be read back as a URI, or a
|
||||
/// handed-over secret key is not text. Neither is the caller's fault.
|
||||
pub(crate) fn from_gateway(
|
||||
context: &RequestContextView,
|
||||
server_ctx: &Arc<ServerContextSlot>,
|
||||
signature: VerifiedSignature,
|
||||
) -> S3Result<Self> {
|
||||
let uri = request_uri(context)?;
|
||||
let mut headers = HeaderMap::new();
|
||||
for (name, value) in context.headers().iter_raw() {
|
||||
headers.append(name.clone(), value.clone());
|
||||
}
|
||||
if !headers.contains_key(http::header::HOST)
|
||||
&& let Some(authority) = context.target_authority()
|
||||
{
|
||||
let host =
|
||||
HeaderValue::from_str(authority).map_err(|_| unreadable("the accepted authority is not a header value"))?;
|
||||
headers.insert(http::header::HOST, host);
|
||||
}
|
||||
let credentials = match context.principal() {
|
||||
Some(principal) => {
|
||||
let secret_key = principal
|
||||
.secret_key_from_authenticator_lookup()
|
||||
.map(|secret| secret.expose_secret());
|
||||
Some(request_credentials(
|
||||
principal.access_key_id(),
|
||||
secret_key,
|
||||
get_session_token(&uri, &headers),
|
||||
)?)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let mut extensions = Extensions::new();
|
||||
let transport = context.transport_extensions();
|
||||
copy_payload::<Option<RemoteAddr>>(transport, &mut extensions);
|
||||
copy_payload::<ClientInfo>(transport, &mut extensions);
|
||||
copy_payload::<NativeTlsTransport>(transport, &mut extensions);
|
||||
copy_payload::<RequestContext>(transport, &mut extensions);
|
||||
copy_payload::<BodyReadControl>(transport, &mut extensions);
|
||||
extensions.insert(Arc::clone(server_ctx));
|
||||
AccessMarkers::new(context.operation(), signature).install(&mut extensions);
|
||||
|
||||
let scope = context.verified_scope();
|
||||
let head = RequestHead {
|
||||
method: context.method().clone(),
|
||||
uri,
|
||||
headers,
|
||||
credentials,
|
||||
region: scope.map(|scope| scope.region().to_owned()),
|
||||
service: scope.map(|scope| scope.service().to_owned()),
|
||||
};
|
||||
Ok(Self::from_parts(head, extensions))
|
||||
}
|
||||
}
|
||||
|
||||
/// The request target as it arrived: the raw path and query, with the scheme and authority when
|
||||
/// the transport carried them (an absolute-form target, or HTTP/2).
|
||||
fn request_uri(context: &RequestContextView) -> S3Result<Uri> {
|
||||
let path_and_query = match context.raw_query() {
|
||||
"" => context.raw_path().to_owned(),
|
||||
query => format!("{}?{query}", context.raw_path()),
|
||||
};
|
||||
let builder = match (context.target_scheme(), context.target_authority()) {
|
||||
(Some(scheme), Some(authority)) => Uri::builder().scheme(scheme).authority(authority),
|
||||
_ => Uri::builder(),
|
||||
};
|
||||
builder
|
||||
.path_and_query(path_and_query)
|
||||
.build()
|
||||
.map_err(|_| unreadable("the accepted request target is not a URI"))
|
||||
}
|
||||
|
||||
/// The caller's credentials as the app layer holds them. `secret_key` is what the gateway handed
|
||||
/// over, which is nothing unless the operation opted in.
|
||||
fn request_credentials(access_key: &str, secret_key: Option<&[u8]>, session_token: Option<&str>) -> S3Result<RequestCredentials> {
|
||||
let session_token = session_token.map(str::to_owned);
|
||||
let Some(secret_key) = secret_key else {
|
||||
return Ok(RequestCredentials::without_secret_key(access_key.to_owned(), session_token));
|
||||
};
|
||||
let secret_key = std::str::from_utf8(secret_key).map_err(|_| unreadable("the caller's secret key is not text"))?;
|
||||
// The gateway refuses a credential whose secret is empty, so a handed-over secret is never
|
||||
// empty. Should one arrive, the request is refused rather than served with a secret that
|
||||
// every stored secret would have to be compared against.
|
||||
RequestCredentials::new(access_key.to_owned(), secret_key.to_owned(), session_token)
|
||||
.map_err(|_| unreadable("the caller's secret key is empty"))
|
||||
}
|
||||
|
||||
/// A request fact the edge cannot carry, which is never the caller's fault.
|
||||
fn unreadable(message: &'static str) -> S3Error {
|
||||
S3Error::with_message(S3ErrorCode::InternalError, message)
|
||||
}
|
||||
|
||||
/// Copies the payload of type `T` a layer in front of the gateway attached, when one did.
|
||||
fn copy_payload<T: Clone + Send + Sync + 'static>(transport: &TransportExtensions, extensions: &mut Extensions) {
|
||||
if let Some(payload) = transport.get::<T>() {
|
||||
extensions.insert(payload.clone());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::server::gateway_handlers::access::verified_signature;
|
||||
use crate::server::gateway_handlers::error::handler_error;
|
||||
use crate::server::gateway_handlers::test_support::{CLIENT, PEER, REQUEST_ID, Signed, request_context, signed_request};
|
||||
use bytes::Bytes;
|
||||
use http::{Method, Request, StatusCode, Version};
|
||||
use http_body_util::{BodyExt, Empty};
|
||||
use rustfs_gateway::{
|
||||
Credentials, Handler, HandlerResult, OperationCodec, RegionSet, Req, Resp, ServiceBuilder, SigV4Authenticator,
|
||||
StaticCredentials, allow_when, dto,
|
||||
};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Mutex;
|
||||
|
||||
const SIGNER_ACCESS_KEY: &str = "AKIAENVELOPESIGNER";
|
||||
const SIGNER_SECRET_KEY: &str = "envelope-signer-secret-key-0123456789";
|
||||
|
||||
/// Records the envelope the gateway edge builds for the request it is handed.
|
||||
struct Probe {
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
seen: Mutex<Option<RequestEnvelope>>,
|
||||
}
|
||||
|
||||
impl<O: OperationCodec> Handler<O> for Probe
|
||||
where
|
||||
O::Output: Default,
|
||||
{
|
||||
async fn call(&self, request: Req<O>) -> HandlerResult<O> {
|
||||
let context = request.context();
|
||||
let envelope =
|
||||
RequestEnvelope::from_gateway(context, &self.server_ctx, verified_signature(context)).map_err(handler_error)?;
|
||||
*self.seen.lock().expect("probe lock") = Some(envelope);
|
||||
Ok(Resp::new(O::Output::default()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends `request` through a gateway whose only handler is the probe, registered for `O`, and
|
||||
/// returns what the probe built.
|
||||
async fn envelope_of<O: OperationCodec>(
|
||||
request: Request<Empty<Bytes>>,
|
||||
server_ctx: &Arc<ServerContextSlot>,
|
||||
) -> RequestEnvelope
|
||||
where
|
||||
O::Output: Default,
|
||||
{
|
||||
let probe = Arc::new(Probe {
|
||||
server_ctx: Arc::clone(server_ctx),
|
||||
seen: Mutex::new(None),
|
||||
});
|
||||
let regions = RegionSet::new(["us-east-1"]).expect("a region");
|
||||
let credentials = StaticCredentials::new()
|
||||
.with(Credentials::new(SIGNER_ACCESS_KEY, SIGNER_SECRET_KEY.as_bytes()).expect("a valid key"));
|
||||
let service = ServiceBuilder::new()
|
||||
.rustfs_profile()
|
||||
.authenticator(SigV4Authenticator::new(Arc::new(credentials), regions).rustfs_profile())
|
||||
.authorizer(allow_when(|request| request.operation == O::NAME))
|
||||
.register::<O, _>(Arc::clone(&probe))
|
||||
.build()
|
||||
.expect("the probe service assembles");
|
||||
|
||||
let response = service.call(request).await;
|
||||
let status = response.status();
|
||||
if status != StatusCode::OK {
|
||||
let body = response
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.map(|body| body.to_bytes())
|
||||
.unwrap_or_default();
|
||||
panic!("the probe must be reached: {status} {}", String::from_utf8_lossy(&body));
|
||||
}
|
||||
probe
|
||||
.seen
|
||||
.lock()
|
||||
.expect("probe lock")
|
||||
.take()
|
||||
.expect("the probe saw the request")
|
||||
}
|
||||
|
||||
fn list_buckets_request(target: &str) -> http::request::Builder {
|
||||
Request::builder().method(Method::GET).uri(target).header("host", "localhost")
|
||||
}
|
||||
|
||||
fn signed_list_buckets(form: Signed) -> Request<Empty<Bytes>> {
|
||||
signed_request(form, Method::GET, "/", "", SIGNER_ACCESS_KEY, SIGNER_SECRET_KEY)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn from_gateway_copies_every_fact_of_the_accepted_request() {
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
let request = list_buckets_request("/?prefix=probe")
|
||||
.header("x-envelope-probe", "probe")
|
||||
.extension(Some(RemoteAddr(SocketAddr::from(PEER))))
|
||||
.extension(ClientInfo::direct(SocketAddr::new(IpAddr::from(CLIENT), 443)))
|
||||
.extension(NativeTlsTransport(true))
|
||||
.extension(request_context(REQUEST_ID))
|
||||
.extension(BodyReadControl::default())
|
||||
.body(Empty::new())
|
||||
.expect("static request");
|
||||
|
||||
let envelope = envelope_of::<dto::ListBuckets>(request, &server_ctx).await;
|
||||
|
||||
assert_eq!(envelope.method(), Method::GET);
|
||||
assert_eq!(envelope.uri().path(), "/");
|
||||
assert_eq!(envelope.uri().query(), Some("prefix=probe"));
|
||||
assert_eq!(envelope.headers().get("x-envelope-probe"), Some(&HeaderValue::from_static("probe")));
|
||||
assert_eq!(envelope.remote_addr(), Some(SocketAddr::from(PEER)));
|
||||
assert_eq!(envelope.client_info().map(|info| info.real_ip), Some(IpAddr::from(CLIENT)));
|
||||
assert_eq!(envelope.native_tls_transport(), Some(true));
|
||||
assert_eq!(envelope.request_context().map(|context| context.request_id.as_str()), Some(REQUEST_ID));
|
||||
assert!(envelope.body_read_control().is_some(), "body_read_control");
|
||||
assert!(
|
||||
envelope.server_context().is_some_and(|slot| Arc::ptr_eq(slot, &server_ctx)),
|
||||
"server_context"
|
||||
);
|
||||
assert!(envelope.residual_extensions().is_empty(), "every copied payload is a named field");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn from_gateway_leaves_absent_payloads_none_and_an_anonymous_caller_unmarked() {
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
let request = list_buckets_request("/").body(Empty::new()).expect("static request");
|
||||
|
||||
let envelope = envelope_of::<dto::ListBuckets>(request, &server_ctx).await;
|
||||
|
||||
assert!(envelope.credentials().is_none(), "credentials");
|
||||
assert_eq!(envelope.region(), None, "region");
|
||||
assert_eq!(envelope.service(), None, "service");
|
||||
assert_eq!(envelope.remote_addr(), None, "remote_addr");
|
||||
assert!(envelope.client_info().is_none(), "client_info");
|
||||
assert_eq!(envelope.native_tls_transport(), None, "native_tls_transport");
|
||||
assert!(envelope.request_context().is_none(), "request_context");
|
||||
assert!(envelope.body_read_control().is_none(), "body_read_control");
|
||||
assert!(envelope.verified_sigv4().is_none(), "verified_sigv4");
|
||||
assert!(envelope.verified_presigned().is_none(), "verified_presigned");
|
||||
assert!(envelope.multipart_staging_operation().is_none(), "ListBuckets stages no upload");
|
||||
assert!(envelope.req_info().is_none(), "the access phase installs req_info, not the edge");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn from_gateway_treats_a_peerless_remote_addr_as_absent() {
|
||||
let server_ctx = ServerContextSlot::new();
|
||||
let request = list_buckets_request("/")
|
||||
.extension(None::<RemoteAddr>)
|
||||
.body(Empty::new())
|
||||
.expect("static request");
|
||||
|
||||
let envelope = envelope_of::<dto::ListBuckets>(request, &server_ctx).await;
|
||||
|
||||
assert_eq!(envelope.remote_addr(), None);
|
||||
assert!(envelope.residual_extensions().is_empty(), "the peerless marker is consumed, not kept");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_header_signature_marks_sigv4_and_not_presigned() {
|
||||
let envelope = envelope_of::<dto::ListBuckets>(signed_list_buckets(Signed::Header), &ServerContextSlot::new()).await;
|
||||
|
||||
assert!(envelope.verified_sigv4().is_some(), "verified_sigv4");
|
||||
assert!(envelope.verified_presigned().is_none(), "verified_presigned");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_query_signature_marks_both_sigv4_and_presigned() {
|
||||
let envelope = envelope_of::<dto::ListBuckets>(signed_list_buckets(Signed::Query), &ServerContextSlot::new()).await;
|
||||
|
||||
assert!(envelope.verified_sigv4().is_some(), "verified_sigv4");
|
||||
assert!(envelope.verified_presigned().is_some(), "verified_presigned");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_operation_that_stages_an_upload_is_marked_as_staging() {
|
||||
let request = Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri("/bucket/key?uploads")
|
||||
.header("host", "localhost")
|
||||
.body(Empty::new())
|
||||
.expect("static request");
|
||||
|
||||
let envelope = envelope_of::<dto::CreateMultipartUpload>(request, &ServerContextSlot::new()).await;
|
||||
|
||||
assert!(envelope.multipart_staging_operation().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_http2_request_without_a_host_line_gets_one_from_its_authority() {
|
||||
let request = Request::builder()
|
||||
.version(Version::HTTP_2)
|
||||
.method(Method::GET)
|
||||
.uri("http://localhost/")
|
||||
.body(Empty::new())
|
||||
.expect("static request");
|
||||
|
||||
let envelope = envelope_of::<dto::ListBuckets>(request, &ServerContextSlot::new()).await;
|
||||
|
||||
let hosts: Vec<&HeaderValue> = envelope.headers().get_all(http::header::HOST).iter().collect();
|
||||
assert_eq!(hosts, [&HeaderValue::from_static("localhost")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credentials_carry_the_secret_only_when_the_gateway_handed_it_over() {
|
||||
let kept = request_credentials("AKIAENVELOPE", None, Some("token")).expect("credentials");
|
||||
assert_eq!(kept.access_key(), "AKIAENVELOPE");
|
||||
assert!(kept.secret_key().is_none());
|
||||
assert_eq!(kept.session_token(), Some("token"));
|
||||
|
||||
let handed = request_credentials("AKIAENVELOPE", Some(b"envelope-secret"), None).expect("credentials");
|
||||
assert_eq!(handed.secret_key(), Some("envelope-secret"));
|
||||
assert_eq!(handed.session_token(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_handed_over_secret_is_refused_instead_of_kept() {
|
||||
let error = request_credentials("AKIAENVELOPE", Some(b""), None).expect_err("empty");
|
||||
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
/// ListBuckets does not opt in to the caller's secret, so the gateway keeps it: the handler
|
||||
/// sees who signed and no secret, and a reader that needs one is refused instead of being
|
||||
/// handed an empty string to compare with.
|
||||
#[tokio::test]
|
||||
async fn an_operation_that_did_not_opt_in_reaches_the_handler_without_the_caller_s_secret() {
|
||||
let envelope = envelope_of::<dto::ListBuckets>(signed_list_buckets(Signed::Header), &ServerContextSlot::new()).await;
|
||||
|
||||
let credentials = envelope.credentials().expect("a signed request has a caller");
|
||||
assert_eq!(credentials.access_key(), SIGNER_ACCESS_KEY);
|
||||
assert!(credentials.secret_key().is_none(), "the secret stays with the gateway");
|
||||
assert!(credentials.require_secret_key().is_err(), "a reader of the secret fails closed");
|
||||
assert!(!credentials.secret_key_matches(""), "an absent secret equals no stored secret");
|
||||
assert!(!credentials.secret_key_matches(SIGNER_SECRET_KEY));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_secret_that_is_not_text_is_refused_instead_of_dropped() {
|
||||
let error = request_credentials("AKIAENVELOPE", Some(&[0xff, 0xfe]), None).expect_err("not text");
|
||||
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! The one conversion of a RustFS error into the gateway's handler error.
|
||||
//!
|
||||
//! Responsible for: [`handler_error`], called by the gateway backend's one handler and nothing
|
||||
//! else.
|
||||
//! NOT responsible for: choosing an error (the use case's), or writing it (the gateway renders the
|
||||
//! document, the status and the request identifiers).
|
||||
//! Upstream: `rustfs_s3_types::S3Error`. Downstream: `rustfs_gateway::HandlerError`.
|
||||
//!
|
||||
//! The gateway's error carries a code with its status, a message, and response headers from the
|
||||
//! closed set of [`ErrorHeader`], every member of which is read here. An error this function cannot
|
||||
//! carry whole is answered `500 InternalError` rather than sent with a member missing: a `416`
|
||||
//! without its `Content-Range`, or a throttle without its `Retry-After`, tells a client something
|
||||
//! RustFS did not say.
|
||||
|
||||
use http::{HeaderName, HeaderValue};
|
||||
use rustfs_gateway::{ErrorHeader, HandlerError, HttpDate, RedirectTarget, RegionLabel, VersionIdLabel};
|
||||
use rustfs_gateway_types::{ErrorCode, Timestamp, TimestampFormat};
|
||||
use rustfs_s3_types::S3Error;
|
||||
use std::borrow::Cow;
|
||||
|
||||
/// Answered when an error names a code without a status: the raiser of a custom code must set one.
|
||||
const MISSING_STATUS: &str = "the error carries a code without a status";
|
||||
/// Answered when an error carries a response header this edge cannot write.
|
||||
const UNWRITABLE_HEADER: &str = "the error carries a response header the gateway cannot write";
|
||||
|
||||
/// Converts a use-case error into the error the gateway answers with: the same code, status and
|
||||
/// message, and every response header it carries.
|
||||
pub(crate) fn handler_error(error: S3Error) -> HandlerError {
|
||||
let parts = error.into_parts();
|
||||
let Some(status) = parts.status.or_else(|| parts.code.status_code()) else {
|
||||
return HandlerError::internal_error(MISSING_STATUS);
|
||||
};
|
||||
// A declared code compares equal to the custom code of the same spelling and status.
|
||||
let code = ErrorCode::custom(parts.code.as_str().to_owned(), status);
|
||||
let mut mapped = HandlerError::new(code, parts.message.unwrap_or(Cow::Borrowed("")));
|
||||
for (name, value) in parts.headers.iter().flatten() {
|
||||
let Some(header) = error_header(name, value) else {
|
||||
return HandlerError::internal_error(UNWRITABLE_HEADER);
|
||||
};
|
||||
mapped = mapped.with_header(header);
|
||||
}
|
||||
mapped
|
||||
}
|
||||
|
||||
/// The gateway's fact for one error response header, or `None` for a header it cannot write.
|
||||
fn error_header(name: &HeaderName, value: &HeaderValue) -> Option<ErrorHeader> {
|
||||
let value = value.to_str().ok()?;
|
||||
match name.as_str() {
|
||||
"content-range" => {
|
||||
let complete_length = value.strip_prefix("bytes */")?.parse().ok()?;
|
||||
Some(ErrorHeader::UnsatisfiedRange { complete_length })
|
||||
}
|
||||
"retry-after" => Some(ErrorHeader::RetryAfter {
|
||||
seconds: value.parse().ok()?,
|
||||
}),
|
||||
"x-amz-bucket-region" => Some(ErrorHeader::BucketRegion {
|
||||
region: RegionLabel::new(value).ok()?,
|
||||
}),
|
||||
"location" => Some(ErrorHeader::RedirectLocation {
|
||||
target: RedirectTarget::new(value).ok()?,
|
||||
}),
|
||||
"x-amz-delete-marker" => (value == "true").then_some(ErrorHeader::DeleteMarker),
|
||||
"x-amz-version-id" => Some(ErrorHeader::VersionId {
|
||||
version_id: VersionIdLabel::new(value).ok()?,
|
||||
}),
|
||||
"last-modified" => {
|
||||
let seconds = Timestamp::parse(value, TimestampFormat::HttpDate).ok()?.secs();
|
||||
Some(ErrorHeader::LastModified {
|
||||
at: HttpDate::from_unix_seconds(seconds).ok()?,
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::{HeaderMap, StatusCode, header};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
|
||||
fn with_headers(code: S3ErrorCode, headers: &[(HeaderName, &'static str)]) -> HandlerError {
|
||||
let mut map = HeaderMap::new();
|
||||
for (name, value) in headers {
|
||||
map.append(name.clone(), HeaderValue::from_static(value));
|
||||
}
|
||||
let mut error = S3Error::with_message(code, "refused");
|
||||
error.set_headers(map);
|
||||
handler_error(error)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_declared_code_keeps_its_spelling_status_and_message() {
|
||||
let mapped = handler_error(S3Error::with_message(S3ErrorCode::AccessDenied, "Access Denied"));
|
||||
|
||||
assert_eq!(mapped.code(), &ErrorCode::ACCESS_DENIED);
|
||||
assert_eq!(mapped.message(), "Access Denied");
|
||||
assert!(mapped.headers().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_internal_error_stays_a_server_fault() {
|
||||
let mapped = handler_error(S3Error::with_message(S3ErrorCode::InternalError, "Not init"));
|
||||
|
||||
assert_eq!(mapped.code(), &ErrorCode::INTERNAL_ERROR);
|
||||
assert_eq!(mapped.message(), "Not init");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_explicit_status_overrides_the_declared_one() {
|
||||
let mut error = S3Error::with_message(S3ErrorCode::AccessDenied, "locked");
|
||||
error.set_status_code(StatusCode::CONFLICT);
|
||||
|
||||
let mapped = handler_error(error);
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "AccessDenied");
|
||||
assert_eq!(mapped.code().default_status(), StatusCode::CONFLICT);
|
||||
assert_ne!(mapped.code(), &ErrorCode::ACCESS_DENIED, "a different status is a different code");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_custom_code_keeps_the_status_its_raiser_set() {
|
||||
let mut error = S3Error::with_message(S3ErrorCode::Custom("RustFsOnly".into()), "custom");
|
||||
error.set_status_code(StatusCode::SERVICE_UNAVAILABLE);
|
||||
|
||||
let mapped = handler_error(error);
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "RustFsOnly");
|
||||
assert_eq!(mapped.code().default_status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(mapped.message(), "custom");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_custom_code_without_a_status_is_a_server_fault_not_a_guess() {
|
||||
let mapped = handler_error(S3Error::with_message(S3ErrorCode::Custom("RustFsOnly".into()), "custom"));
|
||||
|
||||
assert_eq!(mapped.code(), &ErrorCode::INTERNAL_ERROR);
|
||||
assert_eq!(mapped.message(), MISSING_STATUS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_message_is_not_invented() {
|
||||
let mapped = handler_error(S3Error::new(S3ErrorCode::AccessDenied));
|
||||
|
||||
assert_eq!(mapped.code().as_str(), "AccessDenied");
|
||||
assert_eq!(mapped.message(), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_header_of_the_closed_set_is_carried() {
|
||||
let mapped = with_headers(
|
||||
S3ErrorCode::InvalidRange,
|
||||
&[
|
||||
(header::CONTENT_RANGE, "bytes */42"),
|
||||
(header::RETRY_AFTER, "7"),
|
||||
(HeaderName::from_static("x-amz-bucket-region"), "eu-west-1"),
|
||||
(header::LOCATION, "https://bucket.s3.eu-west-1.example.test/"),
|
||||
(HeaderName::from_static("x-amz-delete-marker"), "true"),
|
||||
(HeaderName::from_static("x-amz-version-id"), "null"),
|
||||
(header::LAST_MODIFIED, "Tue, 14 Nov 2023 22:13:20 GMT"),
|
||||
],
|
||||
);
|
||||
|
||||
let carried: Vec<HeaderName> = mapped.headers().iter().map(ErrorHeader::name).collect();
|
||||
assert_eq!(
|
||||
carried,
|
||||
[
|
||||
header::CONTENT_RANGE,
|
||||
header::RETRY_AFTER,
|
||||
HeaderName::from_static("x-amz-bucket-region"),
|
||||
header::LOCATION,
|
||||
HeaderName::from_static("x-amz-delete-marker"),
|
||||
HeaderName::from_static("x-amz-version-id"),
|
||||
header::LAST_MODIFIED,
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
mapped
|
||||
.headers()
|
||||
.contains(&ErrorHeader::UnsatisfiedRange { complete_length: 42 })
|
||||
);
|
||||
assert!(mapped.headers().contains(&ErrorHeader::RetryAfter { seconds: 7 }));
|
||||
let rendered: Vec<String> = mapped.headers().iter().map(ErrorHeader::value).collect();
|
||||
assert_eq!(rendered[6], "Tue, 14 Nov 2023 22:13:20 GMT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_header_outside_the_closed_set_fails_the_whole_error() {
|
||||
let mapped = with_headers(S3ErrorCode::AccessDenied, &[(HeaderName::from_static("x-rustfs-unlisted"), "value")]);
|
||||
|
||||
assert_eq!(mapped.code(), &ErrorCode::INTERNAL_ERROR);
|
||||
assert_eq!(mapped.message(), UNWRITABLE_HEADER);
|
||||
assert!(mapped.headers().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreadable_value_of_a_known_header_fails_the_whole_error() {
|
||||
for (name, value) in [
|
||||
(header::RETRY_AFTER, "soon"),
|
||||
(header::CONTENT_RANGE, "bytes 0-1/42"),
|
||||
(HeaderName::from_static("x-amz-delete-marker"), "false"),
|
||||
(HeaderName::from_static("x-amz-bucket-region"), ""),
|
||||
(header::LAST_MODIFIED, "yesterday"),
|
||||
] {
|
||||
let mapped = with_headers(S3ErrorCode::SlowDown, &[(name.clone(), value)]);
|
||||
|
||||
assert_eq!(mapped.message(), UNWRITABLE_HEADER, "{name}: {value}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch B2 (rustfs/backlog#2765): object listings, multipart listings, and multipart create and abort.
|
||||
//! None is served yet; the batch appends its rows here.
|
||||
|
||||
pub(super) const ROWS: &[super::OpRow] = &[];
|
||||
@@ -0,0 +1,186 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The S3 operations the gateway stack serves, and the one way an operation is served
|
||||
//! (rustfs/backlog#2734).
|
||||
//!
|
||||
//! Responsible for: [`Served`], what one operation supplies; the only `impl Handler` in RustFS,
|
||||
//! which runs every request through the same sequence; and [`OPERATIONS`], the table that both
|
||||
//! registers the handlers and is what the gateway's early gate consults. An operation without a
|
||||
//! row has no handler, and the gateway answers it `501 NotImplemented`.
|
||||
//! NOT responsible for: assembling the service (`server/gateway_stack.rs`), authenticating or
|
||||
//! routing a request (the gateway), or any S3 behaviour (the use cases in `crate::app`).
|
||||
//! Upstream: the gateway's dispatch. Downstream: `crate::app` use cases.
|
||||
//!
|
||||
//! An operation is one `impl Served` and one row of `ROWS` in the family file named after its
|
||||
//! migration batch, the stem `storage/legacy_edge/` uses for the same operations.
|
||||
|
||||
mod access;
|
||||
mod envelope;
|
||||
mod error;
|
||||
|
||||
mod bucket_access;
|
||||
mod bucket_configs;
|
||||
mod bucket_core;
|
||||
mod bucket_replication_lifecycle;
|
||||
mod listing_multipart;
|
||||
mod object_commit;
|
||||
mod object_meta;
|
||||
mod object_misc;
|
||||
mod object_read;
|
||||
mod object_write;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_support;
|
||||
|
||||
use self::error::handler_error;
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::runtime_sources::{AppContext, ServerContextSlot};
|
||||
use rustfs_gateway::{Handler, HandlerResult, OperationCodec, Req, Resp, ServiceBuilder};
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_s3_types::S3Result;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// The backend every gateway operation is served by: the server whose use cases answer.
|
||||
pub(crate) struct GatewayBackend {
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
}
|
||||
|
||||
impl GatewayBackend {
|
||||
pub(crate) fn new(server_ctx: Arc<ServerContextSlot>) -> Self {
|
||||
Self { server_ctx }
|
||||
}
|
||||
|
||||
/// The fixed sequence: the shared access phase, the operation's own authorization, the count,
|
||||
/// the use case. Nothing is skipped and nothing is reordered per operation.
|
||||
async fn serve<O: Served>(&self, request: Req<O>) -> S3Result<Resp<O>> {
|
||||
let mut envelope = access::admit(&self.server_ctx, request.context()).await?;
|
||||
O::authorize(&mut envelope, &request).await?;
|
||||
record_s3_op(O::METRIC);
|
||||
O::execute(self.server_ctx.app_context(), envelope, request).await
|
||||
}
|
||||
}
|
||||
|
||||
/// One S3 operation the gateway stack serves.
|
||||
pub(crate) trait Served: OperationCodec + Sized {
|
||||
/// The label the S3 request counter counts the operation under, the legacy stack's.
|
||||
const METRIC: S3Operation;
|
||||
|
||||
/// The operation's own authorization, after the shared access phase installed the caller: the
|
||||
/// owned form of its legacy access hook, calling the function that hook calls. There is no
|
||||
/// default, so an operation cannot be served without stating what it authorizes.
|
||||
fn authorize(envelope: &mut RequestEnvelope, request: &Req<Self>) -> impl Future<Output = S3Result<()>> + Send;
|
||||
|
||||
/// The use case, with the envelope the access phase completed.
|
||||
fn execute(
|
||||
context: Option<Arc<AppContext>>,
|
||||
envelope: RequestEnvelope,
|
||||
request: Req<Self>,
|
||||
) -> impl Future<Output = S3Result<Resp<Self>>> + Send;
|
||||
}
|
||||
|
||||
impl<O: Served> Handler<O> for GatewayBackend {
|
||||
async fn call(&self, request: Req<O>) -> HandlerResult<O> {
|
||||
self.serve(request).await.map_err(handler_error)
|
||||
}
|
||||
}
|
||||
|
||||
/// One served operation: its name and the registration of its handler.
|
||||
pub(crate) struct OpRow {
|
||||
pub(crate) name: &'static str,
|
||||
register: fn(ServiceBuilder, &Arc<GatewayBackend>) -> ServiceBuilder,
|
||||
}
|
||||
|
||||
const fn row<O: Served>() -> OpRow {
|
||||
OpRow {
|
||||
name: O::NAME,
|
||||
register: register_one::<O>,
|
||||
}
|
||||
}
|
||||
|
||||
fn register_one<O: Served>(builder: ServiceBuilder, backend: &Arc<GatewayBackend>) -> ServiceBuilder {
|
||||
builder.register::<O, _>(Arc::clone(backend))
|
||||
}
|
||||
|
||||
/// A table of served operations, one slice per family.
|
||||
pub(crate) type OperationTable = &'static [&'static [OpRow]];
|
||||
|
||||
/// Every operation the gateway stack serves, in the order of the migration batches. A batch
|
||||
/// appends to its own family's `ROWS`; this list does not change.
|
||||
pub(crate) static OPERATIONS: OperationTable = &[
|
||||
bucket_core::ROWS,
|
||||
bucket_configs::ROWS,
|
||||
bucket_access::ROWS,
|
||||
bucket_replication_lifecycle::ROWS,
|
||||
object_meta::ROWS,
|
||||
listing_multipart::ROWS,
|
||||
object_read::ROWS,
|
||||
object_write::ROWS,
|
||||
object_commit::ROWS,
|
||||
object_misc::ROWS,
|
||||
];
|
||||
|
||||
/// The rows of `table`.
|
||||
pub(crate) fn rows(table: OperationTable) -> impl Iterator<Item = &'static OpRow> {
|
||||
table.iter().flat_map(|family| family.iter())
|
||||
}
|
||||
|
||||
/// Whether `table` serves `operation`.
|
||||
pub(crate) fn serves(table: OperationTable, operation: &str) -> bool {
|
||||
rows(table).any(|row| row.name == operation)
|
||||
}
|
||||
|
||||
/// Registers every operation of `table` on `backend`.
|
||||
pub(crate) fn register(table: OperationTable, builder: ServiceBuilder, backend: &Arc<GatewayBackend>) -> ServiceBuilder {
|
||||
rows(table).fold(builder, |builder, row| (row.register)(builder, backend))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_gateway::OperationSet;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
/// Every operation is served by exactly one row, and the gateway knows each one; every other
|
||||
/// operation the gateway knows has no row, so the early gate refuses it and dispatch has no
|
||||
/// handler for it.
|
||||
#[test]
|
||||
fn every_operation_the_gateway_knows_is_served_by_one_row_or_by_none() {
|
||||
let names: Vec<&str> = rows(OPERATIONS).map(|row| row.name).collect();
|
||||
let served: BTreeSet<&str> = names.iter().copied().collect();
|
||||
let known: BTreeSet<&str> = OperationSet::aws_full().names().collect();
|
||||
|
||||
assert_eq!(names.len(), served.len(), "an operation has two rows: {names:?}");
|
||||
assert!(!served.is_empty(), "the table serves nothing");
|
||||
assert!(served.is_subset(&known), "a row names an unknown operation: {:?}", &served - &known);
|
||||
for operation in &known {
|
||||
assert_eq!(serves(OPERATIONS, operation), served.contains(operation), "{operation}");
|
||||
}
|
||||
assert!(
|
||||
known.iter().any(|operation| !serves(OPERATIONS, operation)),
|
||||
"the probe needs an unserved operation"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registration_registers_exactly_the_rows() {
|
||||
let backend = Arc::new(GatewayBackend::new(ServerContextSlot::new()));
|
||||
|
||||
let registered: BTreeSet<&str> = register(OPERATIONS, ServiceBuilder::new(), &backend).registered().collect();
|
||||
|
||||
assert_eq!(registered, rows(OPERATIONS).map(|row| row.name).collect());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch E1 (rustfs/backlog#2765): CopyObject, UploadPartCopy and CompleteMultipartUpload.
|
||||
//! None is served yet; the batch appends its rows here.
|
||||
|
||||
pub(super) const ROWS: &[super::OpRow] = &[];
|
||||
@@ -0,0 +1,18 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch B1 (rustfs/backlog#2765): object ACL, tagging, legal hold, retention and torrent.
|
||||
//! None is served yet; the batch appends its rows here.
|
||||
|
||||
pub(super) const ROWS: &[super::OpRow] = &[];
|
||||
@@ -0,0 +1,18 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch E2 (rustfs/backlog#2765): DeleteObject, DeleteObjects, RestoreObject and SelectObjectContent.
|
||||
//! None is served yet; the batch appends its rows here.
|
||||
|
||||
pub(super) const ROWS: &[super::OpRow] = &[];
|
||||
@@ -0,0 +1,18 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch C (rustfs/backlog#2765): GetObject, HeadObject and GetObjectAttributes.
|
||||
//! None is served yet; the batch appends its rows here.
|
||||
|
||||
pub(super) const ROWS: &[super::OpRow] = &[];
|
||||
@@ -0,0 +1,18 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway operations of migration batch D (rustfs/backlog#2765): PutObject, UploadPart and PostObject.
|
||||
//! None is served yet; the batch appends its rows here.
|
||||
|
||||
pub(super) const ROWS: &[super::OpRow] = &[];
|
||||
@@ -0,0 +1,89 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The fixtures the gateway stack's unit tests share: the facts RustFS's layers attach, and a
|
||||
//! request signed the way a client signs it.
|
||||
|
||||
use crate::storage_api::server::http::request_context::RequestContext;
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderValue, Method, Request};
|
||||
use http_body_util::Empty;
|
||||
use rustfs_gateway::sig::{AmzDate, PayloadMode, SigService, SigV4Signer, SigningCredentials, SigningRequest, SigningScope};
|
||||
use rustfs_gateway::{Limits, WireRequest};
|
||||
use std::time::Instant;
|
||||
|
||||
pub(crate) const PEER: ([u8; 4], u16) = ([10, 0, 0, 7], 41000);
|
||||
pub(crate) const CLIENT: [u8; 4] = [203, 0, 113, 9];
|
||||
pub(crate) const REQUEST_ID: &str = "7c9e6679-7425-40de-944b-e07fc1f90ae7";
|
||||
|
||||
/// The context RustFS's request layer attaches, naming the request `request_id`.
|
||||
pub(crate) fn request_context(request_id: &str) -> RequestContext {
|
||||
RequestContext {
|
||||
request_id: request_id.to_owned(),
|
||||
x_amz_request_id: request_id.to_owned(),
|
||||
trace_id: None,
|
||||
span_id: None,
|
||||
start_time: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a signature travels.
|
||||
#[derive(Clone, Copy)]
|
||||
pub(crate) enum Signed {
|
||||
Header,
|
||||
Query,
|
||||
}
|
||||
|
||||
/// `method path?query` to `localhost`, signed now for `access_key` and `secret_key`.
|
||||
pub(crate) fn signed_request(
|
||||
form: Signed,
|
||||
method: Method,
|
||||
path: &str,
|
||||
query: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Request<Empty<Bytes>> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(http::header::HOST, HeaderValue::from_static("localhost"));
|
||||
let probe = Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(path)
|
||||
.header(http::header::HOST, "localhost")
|
||||
.body(Bytes::new())
|
||||
.expect("static request");
|
||||
let accepted = WireRequest::accept(probe, &Limits::default()).expect("an acceptable host");
|
||||
let stamp = AmzDate::parse(&chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string()).expect("a stamp");
|
||||
let scope = SigningScope::new(stamp.day(), "us-east-1", SigService::S3).expect("a scope");
|
||||
let credentials = SigningCredentials::new(access_key, secret_key.as_bytes()).expect("credentials");
|
||||
let mut signer = SigV4Signer::new(credentials, scope);
|
||||
let signing = SigningRequest::new(
|
||||
&method,
|
||||
path,
|
||||
query,
|
||||
&headers,
|
||||
accepted.host().raw_for_signing(),
|
||||
PayloadMode::Unsigned,
|
||||
stamp,
|
||||
);
|
||||
let signed = match form {
|
||||
Signed::Header => signer.sign_headers(&signing),
|
||||
Signed::Query => signer.presign(&signing, 300),
|
||||
}
|
||||
.expect("a signable request");
|
||||
let mut builder = Request::builder().method(method).uri(signed.target());
|
||||
for (name, value) in signed.headers() {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
builder.body(Empty::new()).expect("a request")
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! The gateway S3 stack of one server: its assembly, and the facts the host hands every request
|
||||
//! (rustfs/backlog#2734 T2.5).
|
||||
//!
|
||||
//! Responsible for: [`GatewayStack::assemble`], the one place the `rustfs-gateway` service is
|
||||
//! built for RustFS — the RustFS profile, signature verification against RustFS IAM, the early
|
||||
//! gate, explicit deadlines, and the registered handlers — and [`GatewayStack::call`], which
|
||||
//! attaches the host's facts to a request before the gateway accepts it.
|
||||
//! NOT responsible for: choosing the stack (`server/stack.rs`), the middleware around it
|
||||
//! (`server/http.rs`), or any S3 behaviour (`server/gateway_handlers` and the use cases).
|
||||
//! Upstream: `server/http.rs`. Downstream: `rustfs-gateway`, `rustfs-gateway-adapter`.
|
||||
//!
|
||||
//! # What the host keeps
|
||||
//!
|
||||
//! RustFS's own layers stay in front of the gateway and the gateway's counterparts stay off
|
||||
//! (rustfs/backlog#2771): CORS is answered by `ConditionalCorsLayer`, so no CORS source is
|
||||
//! installed and the gateway writes no CORS header; the SSE-C transport layer runs first and the
|
||||
//! gateway's own strict gate stays behind it; and the request identifier is the one
|
||||
//! `ExternalRequestContextLayer` assigned, handed over as the gateway's `HostRequestId`, so an
|
||||
//! answer and RustFS's logs name a request identically.
|
||||
//!
|
||||
//! # Authorization
|
||||
//!
|
||||
//! The gateway's `Authorizer` is only the early gate: it allows an operation the table serves and
|
||||
//! denies every other. Every authorization decision, an anonymous caller's included, is the
|
||||
//! handler's, made by the function the legacy access hook calls (`server/gateway_handlers`), which
|
||||
//! also writes its denial audit record. No policy source and no authorization audit sink is
|
||||
//! installed: nothing reads a policy snapshot, and the gate's only refusal is an operation without
|
||||
//! a handler, which the gateway answers `501 NotImplemented`.
|
||||
|
||||
// The stack is under construction, so a build without debug assertions must not carry it. The
|
||||
// message is checked by `scripts/check_connect_release_boundary.py`.
|
||||
#[cfg(not(debug_assertions))]
|
||||
compile_error!("gateway-stack is restricted to debug builds");
|
||||
|
||||
use crate::runtime_sources::{ServerContextSlot, current_action_credentials};
|
||||
use crate::server::gateway_handlers::{GatewayBackend, OPERATIONS, OperationTable, register, serves};
|
||||
use crate::server::ssec_transport::is_secure_transport;
|
||||
use crate::shared_types::{NativeTlsTransport, RemoteAddr};
|
||||
use crate::storage_api::server::http::request_context::RequestContext;
|
||||
use http::{Extensions, Request, Response};
|
||||
use rustfs_config::RUSTFS_REGION;
|
||||
use rustfs_gateway::{
|
||||
AssemblyError, Authorizer, BoxFuture, ClientAddr, DEFAULT_MAX_BUFFERED_BODY_BYTES, Decision, HandlerDeadlineClass,
|
||||
HandlerDeadlineConfig, MintedTraces, RegionSet, RequestBodyDeadlineConfig, ServiceBuilder, ServiceConfig, SigV4Authenticator,
|
||||
SseConfig, TransportSecurity, decide_with,
|
||||
};
|
||||
use rustfs_gateway_adapter::{
|
||||
IamCredentials, IamKeyLookup, RequestIdCarrier, RootCredentials, RootCredentialsError, RootSource, host_request_id,
|
||||
rustfs_host_resolver,
|
||||
};
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::manager::get_token_signing_key;
|
||||
use rustfs_iam::store::object::ObjectStore;
|
||||
use rustfs_iam::sys::IamSys;
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use rustfs_trusted_proxies::ClientInfo;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
/// What the assembly takes from the server it is built for.
|
||||
pub(crate) struct GatewayAssembly {
|
||||
pub(crate) server_ctx: Arc<ServerContextSlot>,
|
||||
/// The configured root credentials, checked once here. Requests read the live ones.
|
||||
pub(crate) root_access_key: String,
|
||||
pub(crate) root_secret_key: String,
|
||||
/// The configured region; the default region signs when none is configured.
|
||||
pub(crate) region: Option<String>,
|
||||
/// The virtual-hosted-style base domains of the S3 listener, as the legacy stack reads them.
|
||||
pub(crate) server_domains: Vec<String>,
|
||||
/// `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT`: the longest silence between two body reads. Zero
|
||||
/// disables the bound, as it does on the legacy stack.
|
||||
pub(crate) body_read_timeout: Duration,
|
||||
/// Whether Keystone authentication is enabled in configuration (`RUSTFS_KEYSTONE_ENABLE`).
|
||||
pub(crate) keystone_enabled: bool,
|
||||
}
|
||||
|
||||
/// Whether Keystone authentication is switched on in configuration. A Keystone configuration
|
||||
/// that cannot be read counts as switched on: only a readable configuration can say it is off.
|
||||
pub(crate) fn keystone_enabled_in_configuration() -> bool {
|
||||
rustfs_keystone::KeystoneConfig::from_env().map_or(true, |config| config.enable)
|
||||
}
|
||||
|
||||
/// Why the gateway stack could not be assembled. Startup fails with it; the legacy stack is never
|
||||
/// started in its place.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum GatewayAssemblyError {
|
||||
#[error("the root credentials cannot verify signatures on the gateway stack: {0}")]
|
||||
RootCredentials(RootCredentialsError),
|
||||
#[error("region {region:?} cannot be a signing region on the gateway stack")]
|
||||
Region { region: String },
|
||||
#[error("the server domains cannot be served by the gateway stack: {0:?}")]
|
||||
ServerDomains(rustfs_gateway::LegacyDomainError),
|
||||
#[error("the gateway service refused its assembly: {0:?}")]
|
||||
Service(AssemblyError),
|
||||
#[error(
|
||||
"RUSTFS_S3_STACK=gateway cannot be served while RUSTFS_KEYSTONE_ENABLE is on: the gateway admits a request that \
|
||||
carries only X-Auth-Token as anonymous while RustFS reads the Keystone identity behind it; Keystone support on the \
|
||||
gateway stack is rustfs/gateway#1586"
|
||||
)]
|
||||
KeystoneEnabled,
|
||||
}
|
||||
|
||||
/// The assembled gateway service.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct GatewayStack {
|
||||
service: rustfs_gateway::S3Service,
|
||||
}
|
||||
|
||||
impl GatewayStack {
|
||||
/// Assembles the gateway stack for one server.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`GatewayAssemblyError`] naming the input the gateway refused. The secret key is never
|
||||
/// part of it.
|
||||
pub(crate) fn assemble(assembly: GatewayAssembly) -> Result<Self, GatewayAssemblyError> {
|
||||
let iam = Arc::new(ServerIam {
|
||||
server_ctx: Arc::clone(&assembly.server_ctx),
|
||||
});
|
||||
let root = Arc::new(live_root(Arc::clone(&assembly.server_ctx)));
|
||||
Self::assemble_with(assembly, iam, root, early_gate(OPERATIONS))
|
||||
}
|
||||
|
||||
fn assemble_with(
|
||||
assembly: GatewayAssembly,
|
||||
iam: Arc<dyn IamKeyLookup>,
|
||||
root: Arc<dyn RootSource>,
|
||||
authorizer: impl Authorizer,
|
||||
) -> Result<Self, GatewayAssemblyError> {
|
||||
// The pinned gateway admits a request that carries only `X-Auth-Token` as anonymous and
|
||||
// runs no authenticator for it, while the RustFS code behind the gateway reads the
|
||||
// Keystone identity: the request would be authorized as one caller and served as
|
||||
// another. Refused until the gateway authenticates the token (rustfs/gateway#1586).
|
||||
if assembly.keystone_enabled {
|
||||
return Err(GatewayAssemblyError::KeystoneEnabled);
|
||||
}
|
||||
// A root the gateway could not verify a signature with is a startup error, never a
|
||||
// server that silently has no root. Requests read the live root from `root`.
|
||||
RootCredentials::new(&assembly.root_access_key, &assembly.root_secret_key, &assembly.root_secret_key)
|
||||
.map_err(GatewayAssemblyError::RootCredentials)?;
|
||||
let region = assembly.region.as_deref().unwrap_or(RUSTFS_REGION);
|
||||
let regions = RegionSet::new([region]).map_err(|_| GatewayAssemblyError::Region {
|
||||
region: region.to_owned(),
|
||||
})?;
|
||||
let hosts = rustfs_host_resolver(&assembly.server_domains).map_err(GatewayAssemblyError::ServerDomains)?;
|
||||
let backend = Arc::new(GatewayBackend::new(Arc::clone(&assembly.server_ctx)));
|
||||
|
||||
// Every legacy RustFS reading of a request, a credential scope and an answer, in the
|
||||
// gateway's one reviewable preset; its other half is on the authenticator.
|
||||
let credentials = Arc::new(IamCredentials::new(iam, root));
|
||||
let builder = ServiceBuilder::new()
|
||||
.rustfs_profile()
|
||||
.authenticator(SigV4Authenticator::new(credentials, regions).rustfs_profile())
|
||||
.authorizer(authorizer)
|
||||
// The outer SSE-C transport layer answers first; the gateway's own gate stays strict
|
||||
// behind it.
|
||||
.sse_config(SseConfig::strict())
|
||||
.host_resolver(hosts)
|
||||
// The fallback only: `attach_host_facts` hands RustFS's own identifier over.
|
||||
.trace_source(MintedTraces::with_uuid_request_ids());
|
||||
let (builder, _settings) = builder.config(service_config(assembly.body_read_timeout));
|
||||
let service = register(OPERATIONS, builder, &backend)
|
||||
.build()
|
||||
.map_err(GatewayAssemblyError::Service)?;
|
||||
Ok(Self { service })
|
||||
}
|
||||
|
||||
/// Answers one request on the gateway stack, with the host's facts attached.
|
||||
pub(crate) fn call<B>(&self, mut request: Request<B>) -> impl Future<Output = Response<rustfs_gateway::Body>> + Send + 'static
|
||||
where
|
||||
B: http_body::Body + Send + 'static,
|
||||
B::Data: Send,
|
||||
B::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
{
|
||||
attach_host_facts(request.extensions_mut());
|
||||
let mut service = self.service.clone();
|
||||
let answer = tower::Service::call(&mut service, request);
|
||||
async move {
|
||||
match answer.await {
|
||||
Ok(response) => response,
|
||||
Err(never) => match never {},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The early gate over `table`: an operation it serves is allowed on to its handler, which decides;
|
||||
/// every other operation is denied.
|
||||
fn early_gate(table: OperationTable) -> impl Authorizer {
|
||||
decide_with(move |request| {
|
||||
if serves(table, request.operation) {
|
||||
Decision::Allow
|
||||
} else {
|
||||
Decision::Deny
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// The gateway's request settings for RustFS, every deadline stated instead of inherited.
|
||||
///
|
||||
/// No handler runs under a total deadline, object data included: the legacy stack has none, and a
|
||||
/// large transfer must not be cut off for its duration. A committed answer is not bounded either.
|
||||
/// The request body keeps RustFS's own idle bound and no throughput floor.
|
||||
fn service_config(body_read_timeout: Duration) -> ServiceConfig {
|
||||
let handlers = HandlerDeadlineConfig::default()
|
||||
.without_deadline(HandlerDeadlineClass::Standard)
|
||||
.without_deadline(HandlerDeadlineClass::Extended)
|
||||
.without_commit_progress_deadline();
|
||||
ServiceConfig::new(DEFAULT_MAX_BUFFERED_BODY_BYTES)
|
||||
.with_handler_deadlines(handlers)
|
||||
.with_request_body_deadlines(body_deadlines(body_read_timeout))
|
||||
}
|
||||
|
||||
/// RustFS's body read timeout as the gateway's idle deadlines: the same silence is allowed before
|
||||
/// the first byte and between reads, and zero lifts both.
|
||||
fn body_deadlines(read_timeout: Duration) -> RequestBodyDeadlineConfig {
|
||||
RequestBodyDeadlineConfig::new(read_timeout, read_timeout)
|
||||
.unwrap_or(RequestBodyDeadlineConfig::S3.without_idle_deadlines())
|
||||
.without_throughput_floor()
|
||||
}
|
||||
|
||||
/// Attaches what the gateway reads about a request from its host: whether the transport was
|
||||
/// secure, the client address after RustFS's trusted-proxy correction, and RustFS's request
|
||||
/// identifier. Each is derived from what RustFS's own layers already attached; no header is read.
|
||||
fn attach_host_facts(extensions: &mut Extensions) {
|
||||
let client_info = extensions.get::<ClientInfo>();
|
||||
let native_tls = extensions.get::<NativeTlsTransport>().is_some_and(|transport| transport.0);
|
||||
let security = if is_secure_transport(native_tls, client_info) {
|
||||
TransportSecurity::Encrypted
|
||||
} else {
|
||||
TransportSecurity::Plaintext
|
||||
};
|
||||
let client_ip = client_info.map(|info| info.real_ip).or_else(|| {
|
||||
extensions
|
||||
.get::<Option<RemoteAddr>>()
|
||||
.and_then(|peer| peer.as_ref().map(|peer| peer.0.ip()))
|
||||
});
|
||||
// An identifier outside the gateway's alphabet is not handed over; the gateway mints its own.
|
||||
let request_id = extensions.get::<RequestContext>().and_then(host_request_id);
|
||||
|
||||
extensions.insert(security);
|
||||
if let Some(client_ip) = client_ip {
|
||||
extensions.insert(ClientAddr::from_peer(client_ip));
|
||||
}
|
||||
if let Some(request_id) = request_id {
|
||||
extensions.insert(request_id);
|
||||
}
|
||||
}
|
||||
|
||||
/// The identifier the outer layer assigned is the one the gateway is handed.
|
||||
impl RequestIdCarrier for RequestContext {
|
||||
fn request_id(&self) -> &str {
|
||||
&self.request_id
|
||||
}
|
||||
}
|
||||
|
||||
/// The root credentials of one server as they are now, read on every call from where the legacy
|
||||
/// stack reads them, so a root change reaches both stacks at once. `None` while they are not
|
||||
/// initialized or cannot verify a signature, which the adapters answer as unavailable.
|
||||
fn live_root(server_ctx: Arc<ServerContextSlot>) -> impl Fn() -> Option<Arc<RootCredentials>> + Send + Sync + 'static {
|
||||
move || {
|
||||
let root = match server_ctx.app_context() {
|
||||
Some(context) => context.action_credentials().get(),
|
||||
None => current_action_credentials(),
|
||||
}?;
|
||||
// IAM signs STS session tokens with its token-signing key, which is the root secret
|
||||
// until IAM has published one of its own.
|
||||
let signing_key = get_token_signing_key().unwrap_or_else(|| root.secret_key.clone());
|
||||
RootCredentials::new(&root.access_key, &root.secret_key, &signing_key)
|
||||
.ok()
|
||||
.map(Arc::new)
|
||||
}
|
||||
}
|
||||
|
||||
/// The IAM system of one server, looked up when a credential is: IAM is installed after the
|
||||
/// listener starts, so the assembly cannot hold a handle.
|
||||
struct ServerIam {
|
||||
server_ctx: Arc<ServerContextSlot>,
|
||||
}
|
||||
|
||||
impl ServerIam {
|
||||
fn ready(&self) -> Option<Arc<IamSys<ObjectStore>>> {
|
||||
self.server_ctx
|
||||
.app_context()
|
||||
.filter(|context| context.iam().is_ready())
|
||||
.map(|context| context.iam().handle())
|
||||
}
|
||||
}
|
||||
|
||||
impl IamKeyLookup for ServerIam {
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready().is_some()
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
|
||||
Box::pin(async move {
|
||||
let Some(iam) = self.ready() else {
|
||||
return Err(IamError::IamSysNotInitialized);
|
||||
};
|
||||
iam.check_key(access_key).await
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,492 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//! The assembled gateway stack, driven through [`GatewayStack::call`] exactly as `server/http.rs`
|
||||
//! drives it. Requests that need object storage are in `rustfs/tests/embedded_gateway_stack_test.rs`.
|
||||
|
||||
use super::*;
|
||||
use crate::server::gateway_handlers::rows;
|
||||
use crate::server::gateway_handlers::test_support::{CLIENT, PEER, REQUEST_ID, Signed, request_context, signed_request};
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, Method, StatusCode};
|
||||
use http_body_util::{BodyExt, Empty};
|
||||
use rustfs_gateway::{AuthzRequest, ClientFacts, HostRequestId, InputAuthzRequest, InputDecisions, NO_DEADLINE};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Mutex;
|
||||
|
||||
const ROOT_ACCESS_KEY: &str = "rustfsgatewayroot";
|
||||
const ROOT_SECRET_KEY: &str = "rustfs-gateway-root-secret";
|
||||
const EMPTY_SHA256: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
||||
|
||||
fn assembly() -> GatewayAssembly {
|
||||
GatewayAssembly {
|
||||
server_ctx: ServerContextSlot::new(),
|
||||
root_access_key: ROOT_ACCESS_KEY.to_owned(),
|
||||
root_secret_key: ROOT_SECRET_KEY.to_owned(),
|
||||
region: Some("us-east-1".to_owned()),
|
||||
server_domains: Vec::new(),
|
||||
body_read_timeout: Duration::from_secs(300),
|
||||
keystone_enabled: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// An IAM whose readiness and answer are scripted.
|
||||
struct ScriptedIam {
|
||||
ready: bool,
|
||||
}
|
||||
|
||||
impl IamKeyLookup for ScriptedIam {
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
|
||||
Box::pin(async { Ok((None, false)) })
|
||||
}
|
||||
}
|
||||
|
||||
/// A settable live root, standing in for the server's runtime credentials.
|
||||
#[derive(Clone)]
|
||||
struct LiveRoot(Arc<Mutex<Option<Arc<RootCredentials>>>>);
|
||||
|
||||
impl LiveRoot {
|
||||
fn set(&self, root: RootCredentials) {
|
||||
*self.0.lock().expect("root lock") = Some(Arc::new(root));
|
||||
}
|
||||
|
||||
fn clear(&self) {
|
||||
*self.0.lock().expect("root lock") = None;
|
||||
}
|
||||
|
||||
fn source(&self) -> Arc<dyn RootSource> {
|
||||
let cell = Arc::clone(&self.0);
|
||||
Arc::new(move || cell.lock().expect("root lock").clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// A live root holding the test root.
|
||||
fn root() -> LiveRoot {
|
||||
let root = RootCredentials::new(ROOT_ACCESS_KEY, ROOT_SECRET_KEY, ROOT_SECRET_KEY).expect("a usable root");
|
||||
LiveRoot(Arc::new(Mutex::new(Some(Arc::new(root)))))
|
||||
}
|
||||
|
||||
/// The stack as production assembles it, over an IAM that knows no key but the root's.
|
||||
fn stack_over(iam: ScriptedIam) -> GatewayStack {
|
||||
GatewayStack::assemble_with(assembly(), Arc::new(iam), root().source(), early_gate(OPERATIONS)).expect("the stack assembles")
|
||||
}
|
||||
|
||||
fn request(method: Method, target: &str) -> http::request::Builder {
|
||||
Request::builder().method(method).uri(target).header("host", "localhost")
|
||||
}
|
||||
|
||||
/// A well-formed SigV4 header signature for `access_key` whose signature is not the right one.
|
||||
fn signed_with_a_wrong_signature(builder: http::request::Builder, access_key: &str) -> http::request::Builder {
|
||||
let now = jiff::Timestamp::now().strftime("%Y%m%dT%H%M%SZ").to_string();
|
||||
let date = &now[..8];
|
||||
builder
|
||||
.header("x-amz-date", now.as_str())
|
||||
.header("x-amz-content-sha256", EMPTY_SHA256)
|
||||
.header(
|
||||
"authorization",
|
||||
format!(
|
||||
"AWS4-HMAC-SHA256 Credential={access_key}/{date}/us-east-1/s3/aws4_request, \
|
||||
SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature={}",
|
||||
"0".repeat(64)
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
async fn answer(stack: &GatewayStack, builder: http::request::Builder) -> (StatusCode, HeaderMap, String) {
|
||||
answer_request(stack, builder.body(Empty::<Bytes>::new()).expect("static request")).await
|
||||
}
|
||||
|
||||
async fn answer_request(stack: &GatewayStack, request: Request<Empty<Bytes>>) -> (StatusCode, HeaderMap, String) {
|
||||
let response = stack.call(request).await;
|
||||
let (parts, body) = response.into_parts();
|
||||
let body = body.collect().await.expect("the answer is read").to_bytes();
|
||||
(parts.status, parts.headers, String::from_utf8_lossy(&body).into_owned())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_assembled_stack_registers_exactly_the_table() {
|
||||
let stack = GatewayStack::assemble(assembly()).expect("the stack assembles");
|
||||
|
||||
let registered: std::collections::BTreeSet<&str> = stack.service.operations().collect();
|
||||
assert_eq!(registered, rows(OPERATIONS).map(|row| row.name).collect());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unusable_region_fails_the_assembly() {
|
||||
let refused = GatewayStack::assemble(GatewayAssembly {
|
||||
region: Some("Not A Region".to_owned()),
|
||||
..assembly()
|
||||
});
|
||||
|
||||
assert!(matches!(refused, Err(GatewayAssemblyError::Region { .. })), "{:?}", refused.err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unusable_root_access_key_fails_the_assembly_without_naming_the_secret() {
|
||||
let refused = GatewayStack::assemble(GatewayAssembly {
|
||||
root_access_key: String::new(),
|
||||
..assembly()
|
||||
});
|
||||
|
||||
let error = refused.err().expect("an empty access key verifies nothing");
|
||||
assert!(
|
||||
matches!(error, GatewayAssemblyError::RootCredentials(RootCredentialsError::InvalidAccessKey)),
|
||||
"{error:?}"
|
||||
);
|
||||
for rendered in [error.to_string(), format!("{error:?}")] {
|
||||
assert!(!rendered.contains(ROOT_SECRET_KEY), "{rendered}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_or_short_root_secret_fails_the_assembly() {
|
||||
for weak in ["", "1234567"] {
|
||||
let refused = GatewayStack::assemble(GatewayAssembly {
|
||||
root_secret_key: weak.to_owned(),
|
||||
..assembly()
|
||||
});
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
refused.as_ref().err(),
|
||||
Some(GatewayAssemblyError::RootCredentials(RootCredentialsError::WeakSecretKey))
|
||||
),
|
||||
"{} bytes",
|
||||
weak.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_root_change_reaches_the_next_request_and_a_withdrawn_root_signs_nothing() {
|
||||
const ROTATED_ACCESS_KEY: &str = "rotatedgatewayroot";
|
||||
let live = root();
|
||||
let stack =
|
||||
GatewayStack::assemble_with(assembly(), Arc::new(ScriptedIam { ready: true }), live.source(), early_gate(OPERATIONS))
|
||||
.expect("the stack assembles");
|
||||
let signed_as = |access_key: &str| signed_with_a_wrong_signature(request(Method::GET, "/"), access_key);
|
||||
|
||||
let (_, _, known) = answer(&stack, signed_as(ROOT_ACCESS_KEY)).await;
|
||||
assert!(known.contains("<Code>SignatureDoesNotMatch</Code>"), "the root key is known: {known}");
|
||||
|
||||
// The root access key changes: the old one is now just an unknown key, and the new one is
|
||||
// known to the very next request.
|
||||
live.set(RootCredentials::new(ROTATED_ACCESS_KEY, ROOT_SECRET_KEY, ROOT_SECRET_KEY).expect("a usable root"));
|
||||
let (status, _, retired) = answer(&stack, signed_as(ROOT_ACCESS_KEY)).await;
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "{retired}");
|
||||
assert!(retired.contains("<Code>InvalidAccessKeyId</Code>"), "{retired}");
|
||||
let (_, _, rotated) = answer(&stack, signed_as(ROTATED_ACCESS_KEY)).await;
|
||||
assert!(
|
||||
rotated.contains("<Code>SignatureDoesNotMatch</Code>"),
|
||||
"the new root key is known: {rotated}"
|
||||
);
|
||||
|
||||
// Injected: a root source that returns nothing, so the provider returns an error for the
|
||||
// lookup. The key that was root a moment ago verifies nothing, and the fault is answered as
|
||||
// one, `500`, as the legacy stack answers a failed lookup. Ledger entry R-068
|
||||
// (rustfs/backlog#2684): the gateway's RustFS authenticator preset sets the switch of
|
||||
// rustfs/gateway#1585; the assembly sets nothing of its own.
|
||||
live.clear();
|
||||
let (status, _, withdrawn) = answer(&stack, signed_as(ROTATED_ACCESS_KEY)).await;
|
||||
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{withdrawn}");
|
||||
assert!(withdrawn.contains("<Code>InternalError</Code>"), "{withdrawn}");
|
||||
assert!(!withdrawn.contains("ListAllMyBucketsResult"), "{withdrawn}");
|
||||
assert_fault_answer_names_nothing(&withdrawn, ROTATED_ACCESS_KEY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keystone_enabled_fails_the_assembly_and_names_both_settings() {
|
||||
let refused = GatewayStack::assemble(GatewayAssembly {
|
||||
keystone_enabled: true,
|
||||
..assembly()
|
||||
});
|
||||
|
||||
let error = refused.err().expect("the gateway stack is not served with Keystone enabled");
|
||||
assert!(matches!(error, GatewayAssemblyError::KeystoneEnabled), "{error:?}");
|
||||
let message = error.to_string();
|
||||
for named in ["RUSTFS_S3_STACK=gateway", "RUSTFS_KEYSTONE_ENABLE", "rustfs/gateway#1586"] {
|
||||
assert!(message.contains(named), "{named} is missing from: {message}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keystone_is_read_from_configuration_and_an_unreadable_one_counts_as_enabled() {
|
||||
const ENABLE: &str = "RUSTFS_KEYSTONE_ENABLE";
|
||||
const AUTH_URL: &str = "RUSTFS_KEYSTONE_AUTH_URL";
|
||||
|
||||
temp_env::with_vars([(ENABLE, None::<&str>), (AUTH_URL, None)], || {
|
||||
assert!(!keystone_enabled_in_configuration(), "unset is off");
|
||||
});
|
||||
temp_env::with_vars([(ENABLE, Some("false")), (AUTH_URL, None)], || {
|
||||
assert!(!keystone_enabled_in_configuration(), "false is off");
|
||||
});
|
||||
temp_env::with_vars([(ENABLE, Some("true")), (AUTH_URL, Some("http://keystone.example.test:5000"))], || {
|
||||
assert!(keystone_enabled_in_configuration(), "true is on");
|
||||
});
|
||||
temp_env::with_vars([(ENABLE, Some("true")), (AUTH_URL, None)], || {
|
||||
assert!(keystone_enabled_in_configuration(), "on but unreadable is still on");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_handler_runs_under_a_total_deadline_and_the_body_keeps_the_rustfs_idle_bound() {
|
||||
let config = service_config(Duration::from_secs(300));
|
||||
|
||||
assert_eq!(config.handler_deadline(HandlerDeadlineClass::Standard), NO_DEADLINE);
|
||||
assert_eq!(config.handler_deadline(HandlerDeadlineClass::Extended), NO_DEADLINE);
|
||||
assert_eq!(config.commit_progress_deadline(), NO_DEADLINE);
|
||||
assert_eq!(config.request_body_deadlines().read_idle(), Duration::from_secs(300));
|
||||
assert_eq!(config.request_body_deadlines().first_byte(), Duration::from_secs(300));
|
||||
assert_eq!(config.request_body_deadlines().throughput_window(), NO_DEADLINE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_disabled_body_read_timeout_lifts_the_idle_bound_instead_of_arming_zero() {
|
||||
let deadlines = body_deadlines(Duration::ZERO);
|
||||
|
||||
assert_eq!(deadlines.read_idle(), NO_DEADLINE);
|
||||
assert_eq!(deadlines.first_byte(), NO_DEADLINE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_facts_come_from_what_rustfs_layers_attached() {
|
||||
let peer = SocketAddr::from(PEER);
|
||||
let mut extensions = Extensions::new();
|
||||
extensions.insert(Some(RemoteAddr(peer)));
|
||||
extensions.insert(ClientInfo::direct(SocketAddr::new(IpAddr::from(CLIENT), 443)));
|
||||
extensions.insert(NativeTlsTransport(true));
|
||||
extensions.insert(request_context(REQUEST_ID));
|
||||
|
||||
attach_host_facts(&mut extensions);
|
||||
|
||||
assert_eq!(extensions.get::<TransportSecurity>(), Some(&TransportSecurity::Encrypted));
|
||||
assert_eq!(
|
||||
extensions.get::<ClientAddr>().map(|address| address.ip()),
|
||||
Some(IpAddr::from(CLIENT)),
|
||||
"the trusted-proxy-corrected address wins over the socket peer"
|
||||
);
|
||||
assert_eq!(
|
||||
extensions
|
||||
.get::<HostRequestId>()
|
||||
.map(|id| id.request_id().as_str().to_owned()),
|
||||
Some(REQUEST_ID.to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_facts_never_lean_towards_secure_or_invent_an_address() {
|
||||
let mut bare = Extensions::new();
|
||||
attach_host_facts(&mut bare);
|
||||
assert_eq!(bare.get::<TransportSecurity>(), Some(&TransportSecurity::Plaintext));
|
||||
assert!(bare.get::<ClientAddr>().is_none(), "no address is known");
|
||||
assert!(bare.get::<HostRequestId>().is_none(), "no identifier is known");
|
||||
|
||||
let mut plaintext = Extensions::new();
|
||||
plaintext.insert(Some(RemoteAddr(SocketAddr::from(PEER))));
|
||||
plaintext.insert(NativeTlsTransport(false));
|
||||
plaintext.insert(request_context("not a request id"));
|
||||
attach_host_facts(&mut plaintext);
|
||||
assert_eq!(plaintext.get::<TransportSecurity>(), Some(&TransportSecurity::Plaintext));
|
||||
assert_eq!(
|
||||
plaintext.get::<ClientAddr>().map(|address| address.ip()),
|
||||
Some(IpAddr::from(PEER.0)),
|
||||
"without a proxy the socket peer is the client"
|
||||
);
|
||||
assert!(
|
||||
plaintext.get::<HostRequestId>().is_none(),
|
||||
"an identifier outside the alphabet is not handed over"
|
||||
);
|
||||
|
||||
let mut peerless = Extensions::new();
|
||||
peerless.insert(None::<RemoteAddr>);
|
||||
attach_host_facts(&mut peerless);
|
||||
assert!(peerless.get::<ClientAddr>().is_none());
|
||||
}
|
||||
|
||||
/// Records what the gateway hands its authorizer about the client, and refuses.
|
||||
struct RecordingAuthorizer {
|
||||
seen: Arc<Mutex<Option<Option<ClientFacts>>>>,
|
||||
}
|
||||
|
||||
impl Authorizer for RecordingAuthorizer {
|
||||
fn authorize_route<'a>(
|
||||
&'a self,
|
||||
context: &'a rustfs_gateway::RequestContext<'a>,
|
||||
_request: &'a AuthzRequest<'a>,
|
||||
) -> BoxFuture<'a, Decision> {
|
||||
*self.seen.lock().expect("recording lock") = Some(context.client().copied());
|
||||
Box::pin(async { Decision::Deny })
|
||||
}
|
||||
|
||||
fn authorize_input<'a>(
|
||||
&'a self,
|
||||
_context: &'a rustfs_gateway::RequestContext<'a>,
|
||||
request: &'a InputAuthzRequest<'a>,
|
||||
) -> BoxFuture<'a, InputDecisions> {
|
||||
let decisions = request.decide_all(Decision::Deny, |_| Decision::Deny);
|
||||
Box::pin(async move { decisions })
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_authorizer_sees_the_client_facts_the_host_attached() {
|
||||
let seen = Arc::new(Mutex::new(None));
|
||||
let authorizer = RecordingAuthorizer { seen: Arc::clone(&seen) };
|
||||
let stack = GatewayStack::assemble_with(assembly(), Arc::new(ScriptedIam { ready: true }), root().source(), authorizer)
|
||||
.expect("assembles");
|
||||
let proxied = request(Method::GET, "/?prefix=probe")
|
||||
.extension(Some(RemoteAddr(SocketAddr::from(PEER))))
|
||||
.extension(ClientInfo::direct(SocketAddr::new(IpAddr::from(CLIENT), 443)))
|
||||
.extension(NativeTlsTransport(true));
|
||||
|
||||
let (status, _, _) = answer(&stack, proxied).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN);
|
||||
let facts = seen
|
||||
.lock()
|
||||
.expect("recording lock")
|
||||
.take()
|
||||
.expect("the authorizer was asked")
|
||||
.expect("the host attached client facts");
|
||||
assert!(facts.transport_secure, "the listener terminated TLS");
|
||||
assert_eq!(facts.client_ip, Some(IpAddr::from(CLIENT)));
|
||||
assert_eq!(facts.source_ip(), Some(IpAddr::from(CLIENT)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_authorizer_sees_a_cleartext_transport_as_cleartext() {
|
||||
let seen = Arc::new(Mutex::new(None));
|
||||
let authorizer = RecordingAuthorizer { seen: Arc::clone(&seen) };
|
||||
let stack = GatewayStack::assemble_with(assembly(), Arc::new(ScriptedIam { ready: true }), root().source(), authorizer)
|
||||
.expect("assembles");
|
||||
|
||||
let (status, _, _) = answer(&stack, request(Method::GET, "/").extension(NativeTlsTransport(false))).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN);
|
||||
let facts = seen
|
||||
.lock()
|
||||
.expect("recording lock")
|
||||
.take()
|
||||
.expect("the authorizer was asked")
|
||||
.expect("the transport declaration alone makes the facts present");
|
||||
assert!(!facts.transport_secure);
|
||||
assert_eq!(facts.client_ip, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_wrong_signature_is_a_signature_mismatch() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), ROOT_ACCESS_KEY)).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN);
|
||||
assert!(body.contains("<Code>SignatureDoesNotMatch</Code>"), "{body}");
|
||||
assert!(!body.contains(ROOT_SECRET_KEY), "{body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unknown_access_key_is_refused_as_the_caller_s_fault() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), "AKIDUNKNOWN0001")).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "{body}");
|
||||
assert!(body.contains("<Code>InvalidAccessKeyId</Code>"), "{body}");
|
||||
}
|
||||
|
||||
/// The answer to a credential-store fault names neither the key that was looked up, nor a secret,
|
||||
/// nor why the lookup failed.
|
||||
fn assert_fault_answer_names_nothing(body: &str, access_key: &str) {
|
||||
assert!(!body.contains(access_key), "the looked-up key is not echoed: {body}");
|
||||
assert!(!body.contains(ROOT_SECRET_KEY), "{body}");
|
||||
let lowered = body.to_ascii_lowercase();
|
||||
for detail in ["iam", "credential", "lookup", "store", "not ready", "unavailable", "root"] {
|
||||
assert!(!lowered.contains(detail), "the answer says `{detail}`: {body}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Injected: a credential provider that returns an error for the lookup of a non-root key.
|
||||
///
|
||||
/// The fault is answered as one, `500 InternalError`, as the legacy stack answers a failed lookup,
|
||||
/// and not as an unknown key. Up to gateway pin 21283e87 this test observed `403
|
||||
/// InvalidAccessKeyId`; pin 7915d069 changed the answer. Ledger entry R-068 (rustfs/backlog#2684): the gateway's RustFS
|
||||
/// authenticator preset sets the switch of rustfs/gateway#1585; the assembly sets nothing of its
|
||||
/// own. Nothing is served either way.
|
||||
#[tokio::test]
|
||||
async fn a_credential_lookup_that_fails_is_refused_and_never_served() {
|
||||
let stack = stack_over(ScriptedIam { ready: false });
|
||||
|
||||
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), "AKIDUSER0001")).await;
|
||||
|
||||
assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{body}");
|
||||
assert!(body.contains("<Code>InternalError</Code>"), "{body}");
|
||||
assert!(!body.contains("InvalidAccessKeyId"), "a store fault is not an unknown key: {body}");
|
||||
assert_fault_answer_names_nothing(&body, "AKIDUSER0001");
|
||||
assert!(!body.contains("ListAllMyBucketsResult"), "{body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_root_key_verifies_while_iam_is_unavailable() {
|
||||
let stack = stack_over(ScriptedIam { ready: false });
|
||||
|
||||
let (status, _, body) = answer(&stack, signed_with_a_wrong_signature(request(Method::GET, "/"), ROOT_ACCESS_KEY)).await;
|
||||
|
||||
assert_eq!(status, StatusCode::FORBIDDEN, "the root secret is known without IAM: {body}");
|
||||
assert!(body.contains("<Code>SignatureDoesNotMatch</Code>"), "{body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_answer_carries_the_request_identifier_rustfs_assigned() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (_, headers, _) = answer(&stack, request(Method::GET, "/").extension(request_context(REQUEST_ID))).await;
|
||||
|
||||
assert_eq!(headers.get("x-amz-request-id").and_then(|value| value.to_str().ok()), Some(REQUEST_ID));
|
||||
assert_eq!(headers.get("x-request-id").and_then(|value| value.to_str().ok()), Some(REQUEST_ID));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn without_a_rustfs_identifier_the_gateway_mints_its_own() {
|
||||
let stack = stack_over(ScriptedIam { ready: true });
|
||||
|
||||
let (_, headers, _) = answer(&stack, request(Method::GET, "/")).await;
|
||||
|
||||
let minted = headers
|
||||
.get("x-amz-request-id")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.expect("every answer names its request");
|
||||
assert!(!minted.is_empty());
|
||||
}
|
||||
|
||||
/// A correctly signed request for a served operation passes the early gate to its handler; the same
|
||||
/// request is refused at the gate of a stack that serves nothing.
|
||||
#[tokio::test]
|
||||
async fn the_early_gate_allows_only_what_the_table_serves() {
|
||||
static NOTHING: OperationTable = &[];
|
||||
let request = || signed_request(Signed::Header, Method::GET, "/", "", ROOT_ACCESS_KEY, ROOT_SECRET_KEY);
|
||||
let gated = |table| {
|
||||
GatewayStack::assemble_with(assembly(), Arc::new(ScriptedIam { ready: true }), root().source(), early_gate(table))
|
||||
.expect("the stack assembles")
|
||||
};
|
||||
|
||||
let refused = gated(NOTHING).call(request()).await;
|
||||
assert_eq!(refused.status(), StatusCode::FORBIDDEN);
|
||||
|
||||
let (status, _, body) = answer_request(&gated(OPERATIONS), request()).await;
|
||||
assert_ne!(status, StatusCode::FORBIDDEN, "{body}");
|
||||
assert!(!body.contains("AccessDenied"), "{body}");
|
||||
}
|
||||
@@ -18,6 +18,10 @@ use crate::app::object::request_body::{BodyReadControl, ObservedBody};
|
||||
use crate::auth::IAMAuth;
|
||||
use crate::auth_keystone;
|
||||
use crate::config;
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
use crate::server::gateway_stack::{GatewayAssembly, GatewayStack, keystone_enabled_in_configuration};
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
use crate::server::stack::{S3StackService, select_stack};
|
||||
use crate::server::{
|
||||
Http12ShutdownOutcome, ReadinessGateLayer, RemoteAddr, ShutdownHandle,
|
||||
compress::{HttpCompressionConfig, PathAwareHttpCompressionPredicate, PathCategoryInjectionLayer},
|
||||
@@ -259,6 +263,7 @@ const EVENT_HTTP_TRANSPORT_FAILED: &str = "http_transport_failed";
|
||||
const EVENT_SOCKET_FALLBACK: &str = "socket_fallback";
|
||||
const EVENT_HTTP_BIND_FAILED: &str = "http_bind_failed";
|
||||
const EVENT_HTTP_STARTUP_ENDPOINTS: &str = "http_startup_endpoints";
|
||||
const EVENT_S3_STACK_ASSEMBLED: &str = "s3_stack_assembled";
|
||||
const EVENT_HTTP_HOST_ROUTING: &str = "http_host_routing";
|
||||
const EVENT_HTTP_COMPRESSION_STATE: &str = "http_compression_state";
|
||||
const EVENT_API_RATE_LIMIT_STATE: &str = "api_rate_limit_state";
|
||||
@@ -1516,6 +1521,64 @@ fn control_request_body(body: s3s::Body) -> ControlRequestBody {
|
||||
ControlRequestBody(body)
|
||||
}
|
||||
|
||||
/// The S3 service mounted under the middleware. A build without the `gateway-stack` feature mounts
|
||||
/// the legacy service itself.
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
type S3EdgeService = S3Service;
|
||||
|
||||
/// The S3 service mounted under the middleware: the legacy stack or the gateway stack, selected at
|
||||
/// startup (`server/stack.rs`).
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
type S3EdgeService = S3StackService<S3Service, GatewayEdgeService>;
|
||||
|
||||
/// The gateway stack where the legacy service is mounted: the same response and error types, so
|
||||
/// the layers around it do not change with the stack.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
#[derive(Clone)]
|
||||
struct GatewayEdgeService(GatewayStack);
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
impl<B> Service<HttpRequest<B>> for GatewayEdgeService
|
||||
where
|
||||
B: http_body::Body<Data = Bytes> + Send + 'static,
|
||||
B::Error: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
type Response = Response<s3s::Body>;
|
||||
type Error = s3s::HttpError;
|
||||
type Future = Pin<Box<dyn Future<Output = std::result::Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, request: HttpRequest<B>) -> Self::Future {
|
||||
let answer = self.0.call(request);
|
||||
Box::pin(async move { Ok(answer.await.map(s3s::Body::http_body_unsync)) })
|
||||
}
|
||||
}
|
||||
|
||||
/// Selects the S3 stack of this server, once. A gateway stack that cannot be assembled fails
|
||||
/// startup; the legacy stack is never started in its place.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
fn select_s3_stack(
|
||||
config: &config::Config,
|
||||
legacy: S3Service,
|
||||
assembly: impl FnOnce() -> GatewayAssembly,
|
||||
) -> Result<S3EdgeService> {
|
||||
let service = select_stack(config.s3_stack, legacy, || -> Result<GatewayEdgeService> {
|
||||
let stack = GatewayStack::assemble(assembly()).map_err(Error::other)?;
|
||||
Ok(GatewayEdgeService(stack))
|
||||
})?;
|
||||
info!(
|
||||
event = EVENT_S3_STACK_ASSEMBLED,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
s3_stack_assembled = service.assembled(),
|
||||
"S3 stack assembled"
|
||||
);
|
||||
Ok(service)
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ControlBodyDigestService<S> {
|
||||
inner: S,
|
||||
@@ -2508,6 +2571,24 @@ async fn start_http_server_inner(
|
||||
rustfs_config::ENV_HTTP_REQUEST_BODY_READ_TIMEOUT,
|
||||
rustfs_config::DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT,
|
||||
));
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
info!(
|
||||
event = EVENT_S3_STACK_ASSEMBLED,
|
||||
component = LOG_COMPONENT_SERVER,
|
||||
subsystem = LOG_SUBSYSTEM_STARTUP,
|
||||
s3_stack_assembled = "legacy",
|
||||
"S3 stack assembled"
|
||||
);
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
let s3_service = select_s3_stack(config, s3_service, || GatewayAssembly {
|
||||
server_ctx: Arc::clone(&server_ctx),
|
||||
root_access_key: config.access_key.clone(),
|
||||
root_secret_key: config.secret_key.clone(),
|
||||
region: config.region.clone(),
|
||||
server_domains: vh_domain_hosts.to_vec(),
|
||||
body_read_timeout: http_request_body_read_timeout,
|
||||
keystone_enabled: keystone_enabled_in_configuration(),
|
||||
})?;
|
||||
#[cfg(feature = "http3")]
|
||||
let http3_server = if let Some(server_config) = http3_server_config {
|
||||
let h3_s3_service = s3_service.clone();
|
||||
@@ -2950,7 +3031,7 @@ async fn start_http_server_inner(
|
||||
struct ConnectionContext {
|
||||
http_server: Arc<ConnBuilder<HttpTaskExecutor>>,
|
||||
task_owner: HttpTaskExecutor,
|
||||
s3_service: S3Service,
|
||||
s3_service: S3EdgeService,
|
||||
compression_config: HttpCompressionConfig,
|
||||
is_console: bool,
|
||||
/// Whether `RUSTFS_SERVER_DOMAINS` is configured (i.e. s3s virtual-hosted-style routing is active).
|
||||
|
||||
@@ -194,9 +194,6 @@ fn uses_server_owned_s3_request_id<B>(req: &HttpRequest<B>, console_redirect_ena
|
||||
|
||||
let path = req.uri().path();
|
||||
let method = req.method();
|
||||
let is_admin_health_request =
|
||||
(method == Method::GET || method == Method::HEAD) && matches!(path, HEALTH_PREFIX | HEALTH_READY_PATH);
|
||||
let is_profile_request = method == Method::GET && matches!(path, PROFILE_CPU_PATH | PROFILE_MEMORY_PATH);
|
||||
let is_public_health_alias_request = (method == Method::GET || method == Method::HEAD)
|
||||
&& matches!(
|
||||
path,
|
||||
@@ -208,11 +205,7 @@ fn uses_server_owned_s3_request_id<B>(req: &HttpRequest<B>, console_redirect_ena
|
||||
)
|
||||
&& is_public_health_endpoint_request(method, path);
|
||||
|
||||
!(is_admin_path(path)
|
||||
|| is_console_path(path)
|
||||
|| is_admin_health_request
|
||||
|| is_profile_request
|
||||
|| is_public_health_alias_request)
|
||||
!(is_control_plane_route(method, req.uri(), req.headers()) || is_public_health_alias_request)
|
||||
}
|
||||
|
||||
/// Creates a server-owned context and response ID for S3 requests while
|
||||
@@ -869,6 +862,18 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// The routes the admin router answers that are not S3 operations: the admin and table-catalog
|
||||
/// APIs, the console, STS, the public health probes and profiling. The one predicate every place
|
||||
/// that tells them from S3 requests reads.
|
||||
pub(crate) fn is_control_plane_route(method: &Method, uri: &Uri, headers: &HeaderMap) -> bool {
|
||||
let path = uri.path();
|
||||
is_admin_path(path)
|
||||
|| is_console_path(path)
|
||||
|| ((method == Method::GET || method == Method::HEAD) && matches!(path, HEALTH_PREFIX | HEALTH_READY_PATH))
|
||||
|| (method == Method::GET && matches!(path, PROFILE_CPU_PATH | PROFILE_MEMORY_PATH))
|
||||
|| is_sts_query_request(method, uri, headers)
|
||||
}
|
||||
|
||||
pub(crate) fn is_sts_query_request(method: &Method, uri: &Uri, headers: &HeaderMap) -> bool {
|
||||
method == Method::POST
|
||||
&& uri.path() == "/"
|
||||
|
||||
@@ -16,6 +16,10 @@ mod audit;
|
||||
mod compress;
|
||||
pub mod cors;
|
||||
mod event;
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
mod gateway_handlers;
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
mod gateway_stack;
|
||||
mod health;
|
||||
mod http;
|
||||
mod hybrid;
|
||||
@@ -29,6 +33,8 @@ mod runtime;
|
||||
pub(crate) mod runtime_sources;
|
||||
mod service_state;
|
||||
mod ssec_transport;
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
mod stack;
|
||||
pub mod tls_material;
|
||||
pub(crate) mod tls_publish;
|
||||
|
||||
@@ -74,7 +80,15 @@ pub(crate) use health::{
|
||||
pub(crate) use http::HeaderMapCarrier;
|
||||
pub(crate) use http::active_http_requests;
|
||||
pub use layer::s3_http_request_guard;
|
||||
pub(crate) use layer::{RequestContextLayer, is_sts_query_request};
|
||||
pub(crate) use layer::{RequestContextLayer, is_control_plane_route, is_sts_query_request};
|
||||
|
||||
/// The S3 operations the gateway stack serves.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) fn gateway_operations() -> Vec<&'static str> {
|
||||
gateway_handlers::rows(gateway_handlers::OPERATIONS)
|
||||
.map(|row| row.name)
|
||||
.collect()
|
||||
}
|
||||
pub(crate) use module_switch::{
|
||||
MODULE_SWITCHES_SIGNAL_SUBSYSTEM, ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches,
|
||||
current_module_switch_snapshot, refresh_persisted_module_switches_from, refresh_persisted_module_switches_from_store,
|
||||
|
||||
@@ -92,7 +92,7 @@ fn carries_ssec_headers(headers: &HeaderMap) -> bool {
|
||||
/// protocol is only consulted as a second source because the trusted-proxy
|
||||
/// layer has already decided whether the peer is allowed to assert it; a
|
||||
/// request that arrives direct carries no such assertion.
|
||||
fn is_secure_transport(connection_is_tls: bool, client_info: Option<&ClientInfo>) -> bool {
|
||||
pub(crate) fn is_secure_transport(connection_is_tls: bool, client_info: Option<&ClientInfo>) -> bool {
|
||||
if connection_is_tls {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The S3 protocol stack one server runs, selected once at startup (rustfs/backlog#2734 T2.5).
|
||||
//!
|
||||
//! Responsible for: [`S3StackService`], the service mounted where the legacy S3 service used to
|
||||
//! be, and [`select_stack`], the only place a stack is chosen. The whole stack is one or the
|
||||
//! other: no request is routed between stacks by its operation, and a request the gateway stack
|
||||
//! refuses is answered by that refusal and never retried on the legacy stack. With the gateway
|
||||
//! selected, the legacy service receives only what the legacy router claims (the RustFS extension
|
||||
//! routes and the control plane), so no S3 request reaches the legacy S3 implementation.
|
||||
//! NOT responsible for: assembling either stack (`server/http.rs` builds the legacy service,
|
||||
//! `server/gateway_stack.rs` the gateway one), or the middleware around them.
|
||||
//! Upstream: `server/http.rs`. Downstream: the legacy S3 service and the gateway stack.
|
||||
//!
|
||||
//! This file names neither stack's types: it is generic over two services with one response
|
||||
//! type, so `server/http.rs` decides what they are and how their bodies meet.
|
||||
|
||||
use crate::admin::router::claims_route;
|
||||
use crate::config::S3Stack;
|
||||
use http::Request;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll, ready};
|
||||
use tower::Service;
|
||||
|
||||
/// The S3 service of one server: the legacy stack, or the gateway stack with the legacy router
|
||||
/// kept for the routes that are not S3 operations.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct S3StackService<L, G> {
|
||||
legacy: L,
|
||||
gateway: Option<G>,
|
||||
}
|
||||
|
||||
impl<L, G> S3StackService<L, G> {
|
||||
/// The stack this service was built with, as the startup line names it: read from the service
|
||||
/// that answers, never from the configuration that asked for it.
|
||||
pub(crate) fn assembled(&self) -> &'static str {
|
||||
match self.gateway {
|
||||
Some(_) => "gateway",
|
||||
None => "legacy",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the service for the selected stack.
|
||||
///
|
||||
/// `assemble_gateway` runs only when the gateway stack is selected, and its failure is the
|
||||
/// failure of the whole selection: the caller refuses to start.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Whatever `assemble_gateway` refused with. The legacy stack is never substituted.
|
||||
pub(crate) fn select_stack<L, G, E>(
|
||||
stack: S3Stack,
|
||||
legacy: L,
|
||||
assemble_gateway: impl FnOnce() -> Result<G, E>,
|
||||
) -> Result<S3StackService<L, G>, E> {
|
||||
let gateway = match stack {
|
||||
S3Stack::Legacy => None,
|
||||
S3Stack::Gateway => Some(assemble_gateway()?),
|
||||
};
|
||||
Ok(S3StackService { legacy, gateway })
|
||||
}
|
||||
|
||||
impl<L, G, B, R, E> Service<Request<B>> for S3StackService<L, G>
|
||||
where
|
||||
L: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
G: Service<Request<B>, Response = http::Response<R>, Error = E>,
|
||||
L::Future: Send + 'static,
|
||||
G::Future: Send + 'static,
|
||||
{
|
||||
type Response = http::Response<R>;
|
||||
type Error = E;
|
||||
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
if let Some(gateway) = &mut self.gateway {
|
||||
ready!(gateway.poll_ready(cx))?;
|
||||
}
|
||||
self.legacy.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, request: Request<B>) -> Self::Future {
|
||||
match &mut self.gateway {
|
||||
// REMOVED BY T2.11: the gateway serves these routes itself once they are migrated.
|
||||
Some(gateway) if !claims_route(request.method(), request.uri(), request.headers()) => Box::pin(gateway.call(request)),
|
||||
_ => Box::pin(self.legacy.call(request)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::{Method, Response, StatusCode};
|
||||
use std::convert::Infallible;
|
||||
use std::future::{Ready, ready as ready_future};
|
||||
|
||||
/// Answers every request with its own name.
|
||||
#[derive(Clone)]
|
||||
struct Named(&'static str);
|
||||
|
||||
impl Service<Request<()>> for Named {
|
||||
type Response = Response<&'static str>;
|
||||
type Error = Infallible;
|
||||
type Future = Ready<Result<Self::Response, Self::Error>>;
|
||||
|
||||
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn call(&mut self, _request: Request<()>) -> Self::Future {
|
||||
ready_future(Ok(Response::new(self.0)))
|
||||
}
|
||||
}
|
||||
|
||||
fn request(method: Method, target: &str) -> Request<()> {
|
||||
Request::builder()
|
||||
.method(method)
|
||||
.uri(target)
|
||||
.body(())
|
||||
.expect("static request")
|
||||
}
|
||||
|
||||
async fn answered_by(service: &mut S3StackService<Named, Named>, request: Request<()>) -> &'static str {
|
||||
let response = service.call(request).await.expect("infallible");
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
response.into_body()
|
||||
}
|
||||
|
||||
fn gateway_stack() -> S3StackService<Named, Named> {
|
||||
select_stack(S3Stack::Gateway, Named("legacy"), || Ok::<_, Infallible>(Named("gateway"))).expect("assembles")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_gateway_assembly_failure_fails_the_selection_instead_of_falling_back() {
|
||||
let selection = select_stack::<_, Named, _>(S3Stack::Gateway, Named("legacy"), || Err("the assembly was refused"));
|
||||
|
||||
assert_eq!(selection.err(), Some("the assembly was refused"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_selection_never_assembles_a_gateway() {
|
||||
let service = select_stack::<_, Named, &str>(S3Stack::Legacy, Named("legacy"), || {
|
||||
panic!("the legacy stack must not assemble the gateway")
|
||||
})
|
||||
.expect("legacy needs no assembly");
|
||||
|
||||
assert_eq!(service.assembled(), "legacy");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_legacy_stack_answers_every_request_itself() {
|
||||
let mut service = select_stack(S3Stack::Legacy, Named("legacy"), || Ok::<_, Infallible>(Named("gateway")))
|
||||
.expect("legacy needs no assembly");
|
||||
|
||||
for target in ["/", "/bucket/key", "/rustfs/admin/v3/info", "/health"] {
|
||||
assert_eq!(answered_by(&mut service, request(Method::GET, target)).await, "legacy", "{target}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_gateway_stack_answers_every_s3_request_whatever_the_operation() {
|
||||
let mut service = gateway_stack();
|
||||
assert_eq!(service.assembled(), "gateway");
|
||||
|
||||
for (method, target) in [
|
||||
(Method::GET, "/"),
|
||||
(Method::GET, "/bucket/key"),
|
||||
(Method::PUT, "/bucket/key"),
|
||||
(Method::DELETE, "/bucket"),
|
||||
(Method::GET, "/bucket?versions"),
|
||||
(Method::POST, "/bucket/key?uploads"),
|
||||
// Prefixes of the legacy-only routes that are bucket names, not those routes.
|
||||
(Method::GET, "/healthy/key"),
|
||||
(Method::GET, "/rustfs/administrator"),
|
||||
(Method::GET, "/minio/healthcheck"),
|
||||
(Method::GET, "/profile/cpus"),
|
||||
// The control-plane paths under a method or a suffix the legacy router does not answer.
|
||||
(Method::PUT, "/health/x"),
|
||||
(Method::GET, "/health/x"),
|
||||
(Method::PUT, "/profile/cpu"),
|
||||
(Method::DELETE, "/health"),
|
||||
(Method::GET, "/minio/health/x"),
|
||||
(Method::GET, "/minio/health/live"),
|
||||
// Near misses of the RustFS extension routes: a wrong method, an object level.
|
||||
(Method::POST, "/bucket?replication-reset"),
|
||||
(Method::GET, "/bucket/key?events"),
|
||||
(Method::PUT, "/bucket?events"),
|
||||
(Method::GET, "/bucket?lambdaArn=arn"),
|
||||
] {
|
||||
assert_eq!(
|
||||
answered_by(&mut service, request(method.clone(), target)).await,
|
||||
"gateway",
|
||||
"{method} {target}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_gateway_stack_leaves_the_routes_the_legacy_router_claims_to_it() {
|
||||
let mut service = gateway_stack();
|
||||
|
||||
for (method, target) in [
|
||||
(Method::GET, "/rustfs/admin/v3/info"),
|
||||
(Method::PUT, "/rustfs/admin/v3/add-user"),
|
||||
(Method::GET, "/minio/admin/v3/info"),
|
||||
(Method::GET, "/iceberg/v1/config"),
|
||||
(Method::GET, "/health"),
|
||||
(Method::HEAD, "/health/ready"),
|
||||
(Method::GET, "/profile/cpu"),
|
||||
(Method::GET, "/profile/memory"),
|
||||
(Method::GET, "/bucket?replication-metrics"),
|
||||
(Method::PUT, "/bucket?replication-reset"),
|
||||
(Method::GET, "/bucket?events"),
|
||||
(Method::GET, "/?events="),
|
||||
(Method::GET, "/bucket/key?lambdaArn=arn"),
|
||||
] {
|
||||
assert_eq!(
|
||||
answered_by(&mut service, request(method.clone(), target)).await,
|
||||
"legacy",
|
||||
"{method} {target}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_sts_form_post_stays_on_the_legacy_router_and_a_plain_post_does_not() {
|
||||
let mut service = gateway_stack();
|
||||
let sts = Request::builder()
|
||||
.method(Method::POST)
|
||||
.uri("/")
|
||||
.header(http::header::CONTENT_TYPE, "application/x-www-form-urlencoded")
|
||||
.body(())
|
||||
.expect("static request");
|
||||
|
||||
assert_eq!(answered_by(&mut service, sts).await, "legacy");
|
||||
assert_eq!(answered_by(&mut service, request(Method::POST, "/")).await, "gateway");
|
||||
}
|
||||
}
|
||||
@@ -41,7 +41,8 @@ const EVENT_SERVER_STARTING: &str = "server_starting";
|
||||
const EVENT_ACTION_CREDENTIALS_INITIALIZED: &str = "action_credentials_initialized";
|
||||
const EVENT_ACTION_CREDENTIALS_INITIALIZATION_FAILED: &str = "action_credentials_initialization_failed";
|
||||
const DEFAULT_CREDENTIALS_WARNING_MESSAGE: &str = "Detected default root credentials; set RUSTFS_ACCESS_KEY and RUSTFS_SECRET_KEY to non-default values for production deployments";
|
||||
const GATEWAY_STACK_NOT_ASSEMBLED: &str = "gateway stack is not assembled yet (rustfs/backlog#2734 T2.5)";
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
const GATEWAY_STACK_NOT_BUILT: &str = "RUSTFS_S3_STACK=gateway needs a binary built with the `gateway-stack` feature, which no release build enables (rustfs/backlog#2734)";
|
||||
|
||||
pub(crate) struct StartupListenContext {
|
||||
pub(crate) readiness: Arc<GlobalReadiness>,
|
||||
@@ -82,12 +83,7 @@ pub(crate) async fn init_startup_listen_context(
|
||||
config: &Config,
|
||||
instance_ctx: &Arc<InstanceContext>,
|
||||
) -> Result<StartupListenContext> {
|
||||
match config.s3_stack {
|
||||
S3Stack::Legacy => {}
|
||||
// REPLACED BY T2.5
|
||||
// Refuse before any listener or storage exists; never fall back to legacy.
|
||||
S3Stack::Gateway => return Err(Error::new(ErrorKind::Unsupported, GATEWAY_STACK_NOT_ASSEMBLED)),
|
||||
}
|
||||
refuse_unbuilt_stack(config.s3_stack)?;
|
||||
crate::server::init_console_prefix()?;
|
||||
log_sanitized_server_config(config);
|
||||
let readiness = Arc::new(GlobalReadiness::new());
|
||||
@@ -137,6 +133,18 @@ pub(crate) async fn init_startup_listen_context(
|
||||
})
|
||||
}
|
||||
|
||||
/// Refuses a selected S3 stack this binary cannot serve, before any listener or storage exists.
|
||||
/// The selection is never replaced by another stack.
|
||||
fn refuse_unbuilt_stack(stack: S3Stack) -> Result<()> {
|
||||
match stack {
|
||||
S3Stack::Legacy => Ok(()),
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
S3Stack::Gateway => Ok(()),
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
S3Stack::Gateway => Err(Error::new(ErrorKind::Unsupported, GATEWAY_STACK_NOT_BUILT)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_embedded_startup_config(
|
||||
address: String,
|
||||
access_key: String,
|
||||
@@ -166,6 +174,10 @@ pub(crate) async fn prepare_embedded_startup_config(
|
||||
config.secret_key = secret_key.clone();
|
||||
config.region = Some(region.clone());
|
||||
config.console_enable = false;
|
||||
// The stack is selected for the whole process, so an embedded server reads the same variable
|
||||
// the binary does.
|
||||
config.s3_stack = S3Stack::from_env()?;
|
||||
refuse_unbuilt_stack(config.s3_stack)?;
|
||||
|
||||
Ok(EmbeddedStartupConfig {
|
||||
config,
|
||||
@@ -553,6 +565,26 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "gateway-stack"))]
|
||||
#[test]
|
||||
fn a_binary_without_the_gateway_stack_refuses_to_select_it() {
|
||||
let err = super::refuse_unbuilt_stack(crate::config::S3Stack::Gateway).expect_err("the gateway stack is not built");
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::Unsupported);
|
||||
assert!(err.to_string().contains("gateway-stack"), "{err}");
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
#[test]
|
||||
fn a_binary_with_the_gateway_stack_accepts_its_selection() {
|
||||
super::refuse_unbuilt_stack(crate::config::S3Stack::Gateway).expect("the gateway stack is built");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_stack_is_always_selectable() {
|
||||
super::refuse_unbuilt_stack(crate::config::S3Stack::Legacy).expect("legacy is the default stack");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn init_startup_listen_context_rejects_console_s3_port_conflict() {
|
||||
let mut config = Config::new("127.0.0.1:9001", vec!["/tmp/rustfs-data".to_string()]);
|
||||
|
||||
+223
-86
@@ -978,11 +978,13 @@ fn merge_request_object_tag_conditions(
|
||||
/// owned `RequestEnvelope` implements it in the app layer.
|
||||
pub(crate) trait AuthorizationRequest {
|
||||
fn headers(&self) -> &HeaderMap;
|
||||
fn headers_mut(&mut self) -> &mut HeaderMap;
|
||||
fn uri(&self) -> &http::Uri;
|
||||
fn remote_addr(&self) -> Option<std::net::SocketAddr>;
|
||||
fn client_info(&self) -> Option<&ClientInfo>;
|
||||
fn server_context(&self) -> Option<&Arc<ServerContextSlot>>;
|
||||
fn req_info(&self) -> Option<&ReqInfo>;
|
||||
fn req_info_mut(&mut self) -> Option<&mut ReqInfo>;
|
||||
fn internal_object_authorization(&self) -> bool;
|
||||
fn multipart_staging(&self) -> bool;
|
||||
fn staged_multipart_part(&self) -> bool;
|
||||
@@ -1005,6 +1007,9 @@ impl<T> AuthorizationRequest for S3Request<T> {
|
||||
fn headers(&self) -> &HeaderMap {
|
||||
&self.headers
|
||||
}
|
||||
fn headers_mut(&mut self) -> &mut HeaderMap {
|
||||
&mut self.headers
|
||||
}
|
||||
fn uri(&self) -> &http::Uri {
|
||||
&self.uri
|
||||
}
|
||||
@@ -1022,6 +1027,9 @@ impl<T> AuthorizationRequest for S3Request<T> {
|
||||
fn req_info(&self) -> Option<&ReqInfo> {
|
||||
self.extensions.get::<ReqInfo>()
|
||||
}
|
||||
fn req_info_mut(&mut self) -> Option<&mut ReqInfo> {
|
||||
self.extensions.get_mut::<ReqInfo>()
|
||||
}
|
||||
fn internal_object_authorization(&self) -> bool {
|
||||
self.extensions.get::<InternalObjectAuthorization>().is_some()
|
||||
}
|
||||
@@ -1374,6 +1382,182 @@ fn request_iam_store<R: AuthorizationRequest + ?Sized>(req: &R) -> S3Result<Arc<
|
||||
})
|
||||
}
|
||||
|
||||
/// What an edge's signature verification established about a request, as the access phase
|
||||
/// records it for the readers of [`VerifiedSigV4Request`] and [`VerifiedPresignedRequest`].
|
||||
///
|
||||
/// Each edge derives it from its own verdict; the legacy edge from the signature form s3s verified.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub(crate) struct VerifiedSignature {
|
||||
/// A SigV4 signature covered the request line, in the header or the query.
|
||||
pub(crate) sigv4: bool,
|
||||
/// The SigV4 signature was carried in the query.
|
||||
pub(crate) presigned: bool,
|
||||
}
|
||||
|
||||
impl VerifiedSignature {
|
||||
/// The legacy edge's reading: s3s verified the signature form these headers name.
|
||||
pub(crate) fn from_auth_type(auth_type: AuthType) -> Self {
|
||||
Self {
|
||||
sigv4: matches!(auth_type, AuthType::Presigned | AuthType::Signed),
|
||||
presigned: matches!(auth_type, AuthType::Presigned),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The payloads the access phase hands every later reader of a request: the signature markers
|
||||
/// and, for an operation that stages an upload, [`MultipartStagingOperation`].
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) struct AccessMarkers {
|
||||
staging: Option<MultipartStagingOperation>,
|
||||
signature: VerifiedSignature,
|
||||
}
|
||||
|
||||
impl AccessMarkers {
|
||||
/// The markers of the S3 operation named `operation`, signed as `signature` says.
|
||||
pub(crate) fn new(operation: &str, signature: VerifiedSignature) -> Self {
|
||||
Self {
|
||||
staging: MultipartStagingOperation::for_operation(operation),
|
||||
signature,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn install(self, extensions: &mut http::Extensions) {
|
||||
if let Some(staging) = self.staging {
|
||||
extensions.insert(staging);
|
||||
}
|
||||
if self.signature.presigned {
|
||||
extensions.insert(VerifiedPresignedRequest);
|
||||
}
|
||||
if self.signature.sigv4 {
|
||||
extensions.insert(VerifiedSigV4Request);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolves the caller of one request as the access phase of either edge needs it: the credential
|
||||
/// IAM holds for `access_key` (validated against `session_token`), whether it owns the deployment,
|
||||
/// and the request-scoped facts later authorization steps read.
|
||||
///
|
||||
/// `access_key` is the key whose signature the edge verified; `None` is an anonymous request.
|
||||
/// Resolved against `server_ctx` so each embedded server reads its own IAM domain.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// What [`check_key_valid_with_context`] refuses: an unknown, disabled or expired credential, a
|
||||
/// session token that does not belong to it, or an IAM system that is not initialized.
|
||||
pub(crate) async fn resolve_req_info(
|
||||
server_ctx: &Arc<ServerContextSlot>,
|
||||
access_key: Option<&str>,
|
||||
session_token: &str,
|
||||
request_context: Option<RequestContext>,
|
||||
) -> S3Result<ReqInfo> {
|
||||
let app_context = server_ctx.app_context();
|
||||
let (cred, is_owner) = match access_key {
|
||||
Some(access_key) => {
|
||||
let (cred, is_owner) = check_key_valid_with_context(session_token, access_key, app_context.as_deref()).await?;
|
||||
(Some(cred), is_owner)
|
||||
}
|
||||
None => (None, false),
|
||||
};
|
||||
let region = match app_context.as_deref() {
|
||||
Some(context) => context.region().get(),
|
||||
None => runtime_sources::current_region(),
|
||||
};
|
||||
Ok(ReqInfo {
|
||||
cred,
|
||||
is_owner,
|
||||
region,
|
||||
request_context,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// The operation-aware admission every S3 request passes after its caller is resolved and before
|
||||
/// its own authorization: a signed upload-size capability is refused on any operation but the one
|
||||
/// it is scoped to, and the license must be valid.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// `InvalidRequest` for a misplaced capability query, `AccessDenied` for a refused license, and
|
||||
/// `InternalError` when the license cannot be checked.
|
||||
pub(crate) fn admit_operation(
|
||||
headers: &HeaderMap,
|
||||
raw_query: Option<&str>,
|
||||
operation: &str,
|
||||
signature: VerifiedSignature,
|
||||
) -> S3Result<()> {
|
||||
// The size capability is intentionally scoped to the single-object
|
||||
// PutObject operation. Validate this at the operation-aware access
|
||||
// boundary so unsupported GET/HEAD/DELETE/bucket routes cannot silently
|
||||
// ignore a signed capability query.
|
||||
if parse_presigned_put_max_content_length(headers, raw_query, signature.presigned)?.is_some() && operation != "PutObject" {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!("{RUSTFS_MAX_CONTENT_LENGTH_QUERY} is only supported for presigned PutObject"),
|
||||
));
|
||||
}
|
||||
if parse_presigned_multipart_max_total_object_size(headers, raw_query, signature.sigv4)?.is_some()
|
||||
&& operation != "CreateMultipartUpload"
|
||||
{
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!("{RUSTFS_MAX_TOTAL_OBJECT_SIZE_QUERY} is only supported for CreateMultipartUpload"),
|
||||
));
|
||||
}
|
||||
license_check().map_err(|er| match er.kind() {
|
||||
std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"),
|
||||
_ => {
|
||||
tracing::error!("license check failed due to unexpected error: {er}");
|
||||
s3_error!(InternalError, "License validation failed")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod access_phase_tests {
|
||||
use super::{
|
||||
AccessMarkers, AuthType, MultipartStagingOperation, VerifiedPresignedRequest, VerifiedSigV4Request, VerifiedSignature,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn only_a_sigv4_header_or_query_signature_is_marked_verified() {
|
||||
for (auth_type, expected) in [
|
||||
(AuthType::Signed, (true, false)),
|
||||
(AuthType::Presigned, (true, true)),
|
||||
(AuthType::SignedV2, (false, false)),
|
||||
(AuthType::PresignedV2, (false, false)),
|
||||
(AuthType::StreamingSigned, (false, false)),
|
||||
(AuthType::PostPolicy, (false, false)),
|
||||
(AuthType::Anonymous, (false, false)),
|
||||
] {
|
||||
let signature = VerifiedSignature::from_auth_type(auth_type.clone());
|
||||
assert_eq!((signature.sigv4, signature.presigned), expected, "{auth_type:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_marker_is_installed_exactly_when_its_fact_holds() {
|
||||
for (operation, sigv4, presigned) in [
|
||||
("ListBuckets", false, false),
|
||||
("ListBuckets", true, false),
|
||||
("ListBuckets", true, true),
|
||||
("CreateMultipartUpload", false, false),
|
||||
("UploadPart", true, true),
|
||||
("UploadPartCopy", true, false),
|
||||
("CompleteMultipartUpload", true, false),
|
||||
("PutObject", true, true),
|
||||
] {
|
||||
let mut extensions = http::Extensions::new();
|
||||
AccessMarkers::new(operation, VerifiedSignature { sigv4, presigned }).install(&mut extensions);
|
||||
|
||||
let staging = matches!(operation, "CreateMultipartUpload" | "UploadPart" | "UploadPartCopy");
|
||||
assert_eq!(extensions.get::<MultipartStagingOperation>().is_some(), staging, "{operation}");
|
||||
assert_eq!(extensions.get::<VerifiedSigV4Request>().is_some(), sigv4, "{operation}");
|
||||
assert_eq!(extensions.get::<VerifiedPresignedRequest>().is_some(), presigned, "{operation}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ListBucketsIamAuthorization {
|
||||
iam_store: Arc<IamSys<ObjectStore>>,
|
||||
prepared: PreparedIamAuth,
|
||||
@@ -1413,8 +1597,10 @@ impl ListBucketsIamAuthorization {
|
||||
|
||||
/// Prepare the IAM-only authorization used to decide which buckets are visible in ListBuckets.
|
||||
/// Bucket policies are intentionally excluded from this discovery decision, matching MinIO.
|
||||
pub(crate) async fn prepare_list_buckets_iam_authorization<T>(req: &S3Request<T>) -> S3Result<ListBucketsIamAuthorization> {
|
||||
let req_info = req_info_ref(req)?;
|
||||
pub(crate) async fn prepare_list_buckets_iam_authorization<R: AuthorizationRequest + ?Sized>(
|
||||
req: &R,
|
||||
) -> S3Result<ListBucketsIamAuthorization> {
|
||||
let req_info = req.req_info().ok_or_else(missing_req_info_error)?;
|
||||
let Some(cred) = req_info.cred.as_ref() else {
|
||||
return Err(ApiError::access_denied().into());
|
||||
};
|
||||
@@ -1467,8 +1653,8 @@ pub(crate) async fn prepare_list_buckets_iam_authorization<T>(req: &S3Request<T>
|
||||
|
||||
/// Preserve the top-level IAM denial audit emitted before ListBuckets falls back
|
||||
/// to bucket-level visibility checks.
|
||||
pub(crate) fn log_list_buckets_iam_implicit_deny<T>(req: &S3Request<T>) -> S3Result<()> {
|
||||
let req_info = req_info_ref(req)?;
|
||||
pub(crate) fn log_list_buckets_iam_implicit_deny<R: AuthorizationRequest + ?Sized>(req: &R) -> S3Result<()> {
|
||||
let req_info = req.req_info().ok_or_else(missing_req_info_error)?;
|
||||
let denial = DenialContext {
|
||||
quiet: true,
|
||||
suppress_audit: true,
|
||||
@@ -1693,15 +1879,33 @@ impl BulkDenialAudit {
|
||||
}
|
||||
|
||||
/// Authorizes the request based on the action and credentials.
|
||||
pub async fn authorize_request<T>(req: &mut S3Request<T>, action: Action) -> S3Result<()> {
|
||||
pub(crate) async fn authorize_request<R: AuthorizationRequest + ?Sized>(req: &mut R, action: Action) -> S3Result<()> {
|
||||
// Restore the `x-amz-meta-*` user metadata the SigV4 presigners hoist into
|
||||
// the query string. This runs after s3s has verified the signature — its own
|
||||
// verifier rejects an `x-amz-*` header outside `SignedHeaders`, which is
|
||||
// exactly what a hoisted parameter becomes — and before policy evaluation and
|
||||
// the handler read request properties, so every operation sees one request
|
||||
// shape (rustfs/rustfs#9142).
|
||||
crate::auth::hoist_signed_query_params(&mut req.headers, req.uri.query())?;
|
||||
authorize_request_with_location_constraint(req, action, None).await
|
||||
let query = req.uri().query().map(str::to_owned);
|
||||
crate::auth::hoist_signed_query_params(req.headers_mut(), query.as_deref())?;
|
||||
let internal_object_authorization = req.internal_object_authorization();
|
||||
authorize_request_core(req, action, None, internal_object_authorization).await
|
||||
}
|
||||
|
||||
/// Authorizes `action` on `bucket` for either edge: the whole access hook of an operation that
|
||||
/// names one bucket and asks one action, such as HeadBucket.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// `InternalError` when the access phase installed no `ReqInfo`; otherwise what
|
||||
/// [`authorize_request`] refuses.
|
||||
pub(crate) async fn authorize_bucket_request<R: AuthorizationRequest + ?Sized>(
|
||||
req: &mut R,
|
||||
bucket: &str,
|
||||
action: Action,
|
||||
) -> S3Result<()> {
|
||||
req.req_info_mut().ok_or_else(missing_req_info_error)?.bucket = Some(bucket.to_owned());
|
||||
authorize_request(req, action).await
|
||||
}
|
||||
|
||||
/// Authorize the exact tag set selected by CopyObject's source reader before publication.
|
||||
@@ -2693,86 +2897,21 @@ impl S3Access for FS {
|
||||
// second embedded server validates keys against its own root identity
|
||||
// and IAM domain; the slot resolves the process default when it has
|
||||
// not been installed, keeping single-instance behavior unchanged.
|
||||
let app_context = self.server_ctx().app_context();
|
||||
let (cred, is_owner) = if let Some(input_cred) = cx.credentials() {
|
||||
let (cred, is_owner) = check_key_valid_with_context(
|
||||
get_session_token(cx.uri(), cx.headers()).unwrap_or_default(),
|
||||
&input_cred.access_key,
|
||||
app_context.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
(Some(cred), is_owner)
|
||||
} else {
|
||||
(None, false)
|
||||
};
|
||||
|
||||
let access_key = cx.credentials().map(|credentials| credentials.access_key.clone());
|
||||
let session_token = get_session_token(cx.uri(), cx.headers()).unwrap_or_default().to_owned();
|
||||
let request_context = cx.extensions_mut().get::<RequestContext>().cloned();
|
||||
let req_info = resolve_req_info(self.server_ctx(), access_key.as_deref(), &session_token, request_context).await?;
|
||||
|
||||
let region = match app_context.as_deref() {
|
||||
Some(context) => context.region().get(),
|
||||
None => runtime_sources::current_region(),
|
||||
};
|
||||
|
||||
let req_info = ReqInfo {
|
||||
cred,
|
||||
is_owner,
|
||||
region,
|
||||
request_context,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let auth_type = get_request_auth_type_with_query(cx.headers(), cx.uri().query());
|
||||
let verified_presigned = matches!(auth_type, AuthType::Presigned);
|
||||
let verified_sigv4 = matches!(auth_type, AuthType::Presigned | AuthType::Signed);
|
||||
let multipart_staging_operation = MultipartStagingOperation::for_operation(cx.s3_op().name());
|
||||
let signature = VerifiedSignature::from_auth_type(get_request_auth_type_with_query(cx.headers(), cx.uri().query()));
|
||||
let markers = AccessMarkers::new(cx.s3_op().name(), signature);
|
||||
// Publish this server's context slot so downstream data-plane handlers
|
||||
// resolve the same store (backlog#1052 S6).
|
||||
{
|
||||
let ext = cx.extensions_mut();
|
||||
ext.insert(self.server_ctx().clone());
|
||||
ext.insert(req_info);
|
||||
if let Some(operation) = multipart_staging_operation {
|
||||
ext.insert(operation);
|
||||
}
|
||||
if verified_presigned {
|
||||
ext.insert(VerifiedPresignedRequest);
|
||||
}
|
||||
if verified_sigv4 {
|
||||
ext.insert(VerifiedSigV4Request);
|
||||
}
|
||||
}
|
||||
let ext = cx.extensions_mut();
|
||||
ext.insert(self.server_ctx().clone());
|
||||
ext.insert(req_info);
|
||||
markers.install(ext);
|
||||
|
||||
// The size capability is intentionally scoped to the single-object
|
||||
// PutObject operation. Validate this at the operation-aware access
|
||||
// boundary so unsupported GET/HEAD/DELETE/bucket routes cannot silently
|
||||
// ignore a signed capability query.
|
||||
if parse_presigned_put_max_content_length(cx.headers(), cx.uri().query(), verified_presigned)?.is_some()
|
||||
&& cx.s3_op().name() != "PutObject"
|
||||
{
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!("{RUSTFS_MAX_CONTENT_LENGTH_QUERY} is only supported for presigned PutObject"),
|
||||
));
|
||||
}
|
||||
if parse_presigned_multipart_max_total_object_size(cx.headers(), cx.uri().query(), verified_sigv4)?.is_some()
|
||||
&& cx.s3_op().name() != "CreateMultipartUpload"
|
||||
{
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
format!("{RUSTFS_MAX_TOTAL_OBJECT_SIZE_QUERY} is only supported for CreateMultipartUpload"),
|
||||
));
|
||||
}
|
||||
license_check().map_err(|er| match er.kind() {
|
||||
std::io::ErrorKind::PermissionDenied => s3_error!(AccessDenied, "{er}"),
|
||||
_ => {
|
||||
tracing::error!("license check failed due to unexpected error: {er}");
|
||||
s3_error!(InternalError, "License validation failed")
|
||||
}
|
||||
})?;
|
||||
|
||||
// Verify uniformly here? Or verify separately below?
|
||||
|
||||
Ok(())
|
||||
admit_operation(cx.headers(), cx.uri().query(), cx.s3_op().name(), signature)
|
||||
}
|
||||
|
||||
/// Checks whether the CreateBucket request has accesses to the resources.
|
||||
@@ -3536,10 +3675,8 @@ impl S3Access for FS {
|
||||
///
|
||||
/// This method returns `Ok(())` by default.
|
||||
async fn head_bucket(&self, req: &mut S3Request<HeadBucketInput>) -> S3Result<()> {
|
||||
let req_info = ext_req_info_mut(&mut req.extensions)?;
|
||||
req_info.bucket = Some(req.input.bucket.clone());
|
||||
|
||||
authorize_request(req, Action::S3Action(S3Action::ListBucketAction)).await
|
||||
let bucket = req.input.bucket.clone();
|
||||
authorize_bucket_request(req, &bucket, Action::S3Action(S3Action::ListBucketAction)).await
|
||||
}
|
||||
|
||||
/// Checks whether the HeadObject request has accesses to the resources.
|
||||
|
||||
@@ -13,17 +13,21 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::{StorageError, decode_tags_to_map, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::app::RequestEnvelope;
|
||||
use crate::error::ApiErrorDiagnostic;
|
||||
use crate::storage::s3_api;
|
||||
use crate::storage::storage_api::ecfs_consumer::contract::object::ObjectOperations as _;
|
||||
use crate::storage::storage_api::runtime_sources_consumer::{ECStore, runtime_sources};
|
||||
use http::StatusCode;
|
||||
use rustfs_gateway_types::compat::ConversionError;
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_buckets;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_utils::http::headers::{AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER};
|
||||
use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error};
|
||||
use std::collections::HashMap;
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tracing::{debug, error, instrument, warn};
|
||||
use uuid::Uuid;
|
||||
@@ -280,6 +284,33 @@ pub(crate) fn validate_object_lock_configuration_input(input_cfg: &ObjectLockCon
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs a use case that speaks the request envelope and the gateway's types behind this edge: the
|
||||
/// input and the output cross the gateway's compatibility seam, and nothing else is converted.
|
||||
pub(in crate::storage) async fn through_gateway_seam<I, GatewayInput, GatewayOutput, O, F>(
|
||||
req: S3Request<I>,
|
||||
input_from_s3s: fn(I) -> Result<GatewayInput, ConversionError>,
|
||||
output_to_s3s: fn(GatewayOutput) -> Result<O, ConversionError>,
|
||||
use_case: impl FnOnce(RequestEnvelope, GatewayInput) -> F,
|
||||
) -> S3Result<S3Response<O>>
|
||||
where
|
||||
F: Future<Output = rustfs_s3_types::S3Result<GatewayOutput>>,
|
||||
{
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let input = input_from_s3s(req.input).map_err(seam_error)?;
|
||||
let output = use_case(envelope, input).await?;
|
||||
output_to_s3s(output).map(S3Response::new).map_err(seam_error)
|
||||
}
|
||||
|
||||
/// A value this edge and the gateway's types disagree on, answered as a server fault. The seam's
|
||||
/// bucket and key conversions become typed with rustfs/gateway#1610; after that re-pin a name the
|
||||
/// gateway's floor refuses maps here to the client error the legacy edge answers.
|
||||
pub(in crate::storage) fn seam_error(error: ConversionError) -> S3Error {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InternalError,
|
||||
format!("failed to convert through the gateway compatibility seam: {error}"),
|
||||
)
|
||||
}
|
||||
|
||||
// The bodies of the methods below, one file per migration batch (rustfs/backlog#2765).
|
||||
#[path = "legacy_edge/mod.rs"]
|
||||
mod legacy_edge;
|
||||
@@ -531,7 +562,15 @@ impl S3 for FS {
|
||||
// List buckets not associated with a bucket, give it bucket label "*" to denote "all".
|
||||
record_s3_op(S3Operation::ListBuckets);
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_list_buckets(req).await
|
||||
let mut response =
|
||||
through_gateway_seam(req, list_buckets::input_from_s3s, list_buckets::output_to_s3s, |envelope, input| {
|
||||
usecase.execute_list_buckets(envelope, input)
|
||||
})
|
||||
.await?;
|
||||
// The seam writes a listing without buckets as an absent member. This edge has always
|
||||
// answered the element, empty or not, and clients index the answer by it.
|
||||
response.output.buckets.get_or_insert_with(Vec::new);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn list_multipart_uploads(
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
//! former trait method bodies, unchanged; the batch that moves these operations onto the gateway
|
||||
//! owns this file and is the only one that edits it.
|
||||
|
||||
use crate::app::dto_values::stored_bucket_name;
|
||||
use crate::storage::access::bucket_config_mutation_incarnation;
|
||||
use crate::storage::ecfs::*;
|
||||
use crate::storage::helper::audit_only_operation;
|
||||
@@ -33,6 +34,8 @@ use crate::storage::{
|
||||
get_bucket_request_payment_config, get_bucket_website_config, update_bucket_metadata_config_if_incarnation,
|
||||
};
|
||||
use crate::storage::{persisted_from_s3s, persisted_to_s3s};
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::head_bucket;
|
||||
use rustfs_gateway_types::dto::HeadBucketInput as HeadBucketGatewayInput;
|
||||
use rustfs_io_metrics::record_s3_op;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use tracing::instrument;
|
||||
@@ -208,8 +211,16 @@ impl FS {
|
||||
&self,
|
||||
req: S3Request<HeadBucketInput>,
|
||||
) -> S3Result<S3Response<HeadBucketOutput>> {
|
||||
let usecase = s3_api::bucket_usecase_for(self);
|
||||
usecase.execute_head_bucket(req).await
|
||||
record_s3_op(S3Operation::HeadBucket);
|
||||
// Interim until the re-pin onto rustfs/gateway#1610: the seam's input conversion holds the
|
||||
// bucket to the AWS grammar, which refuses names RustFS holds, so the name is taken under the
|
||||
// floor here and only the output crosses the seam.
|
||||
let input = HeadBucketGatewayInput {
|
||||
bucket: stored_bucket_name(&req.input.bucket)?,
|
||||
..Default::default()
|
||||
};
|
||||
let output = s3_api::bucket_usecase_for(self).execute_head_bucket(input).await?;
|
||||
Ok(S3Response::new(head_bucket::output_to_s3s(output).map_err(seam_error)?))
|
||||
}
|
||||
|
||||
pub(in crate::storage::ecfs) async fn put_bucket_accelerate_configuration_edge(
|
||||
|
||||
@@ -13,15 +13,14 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::storage::s3_api::common::rustfs_owner;
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::contract::{
|
||||
bucket::BucketInfo,
|
||||
list::{ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info},
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::contract::list::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
};
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::to_s3s_xml_etag;
|
||||
use percent_encoding::percent_decode_str;
|
||||
use s3s::dto::{
|
||||
Bucket, CommonPrefix, DeleteMarkerEntry, EncodingType, ListBucketsOutput, ListObjectVersionsOutput, ListObjectsOutput,
|
||||
ListObjectsV2Output, Object, ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Timestamp,
|
||||
CommonPrefix, DeleteMarkerEntry, EncodingType, ListObjectVersionsOutput, ListObjectsOutput, ListObjectsV2Output, Object,
|
||||
ObjectStorageClass, ObjectVersion, ObjectVersionStorageClass, Timestamp,
|
||||
};
|
||||
use s3s::{S3Error, S3ErrorCode};
|
||||
use tracing::debug;
|
||||
@@ -126,23 +125,6 @@ pub(crate) fn build_list_objects_output(v2: ListObjectsV2Output, request_marker:
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build_list_buckets_output(bucket_infos: &[BucketInfo]) -> ListBucketsOutput {
|
||||
let buckets: Vec<Bucket> = bucket_infos
|
||||
.iter()
|
||||
.map(|bucket_info| Bucket {
|
||||
creation_date: Some(Timestamp::from(bucket_info.created.unwrap_or(time::OffsetDateTime::UNIX_EPOCH))),
|
||||
name: Some(bucket_info.name.clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
|
||||
ListBucketsOutput {
|
||||
buckets: Some(buckets),
|
||||
owner: Some(rustfs_owner()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_list_object_versions_params(
|
||||
prefix: Option<String>,
|
||||
delimiter: Option<String>,
|
||||
@@ -450,19 +432,16 @@ fn calculate_next_marker(v2: &ListObjectsV2Output) -> Option<String> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
ListObjectVersionsInfo, ListObjectVersionsParams, ListObjectsV2Info, build_list_buckets_output,
|
||||
build_list_object_versions_output, build_list_objects_output, build_list_objects_v2_output,
|
||||
encode_list_objects_v2_continuation_token, parse_list_object_versions_params, parse_list_objects_v2_params,
|
||||
to_s3s_xml_etag,
|
||||
ListObjectVersionsInfo, ListObjectVersionsParams, ListObjectsV2Info, build_list_object_versions_output,
|
||||
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
|
||||
parse_list_object_versions_params, parse_list_objects_v2_params, to_s3s_xml_etag,
|
||||
};
|
||||
use crate::storage::s3_api::common::rustfs_owner;
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::StorageObjectInfo as ObjectInfo;
|
||||
use crate::storage::storage_api::s3_api_consumer::bucket::contract::bucket::BucketInfo;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::{CommonPrefix, EncodingType, ListObjectsV2Output, Object};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
@@ -525,36 +504,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_list_buckets_output_maps_bucket_infos_and_owner() {
|
||||
let bucket_infos = vec![
|
||||
BucketInfo {
|
||||
name: "bucket-a".to_string(),
|
||||
created: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
},
|
||||
BucketInfo {
|
||||
name: "bucket-b".to_string(),
|
||||
created: None,
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
|
||||
let output = build_list_buckets_output(&bucket_infos);
|
||||
let buckets = output.buckets.as_ref().expect("buckets should be present");
|
||||
let owner = output.owner.as_ref().expect("owner should be present");
|
||||
|
||||
assert_eq!(buckets.len(), 2);
|
||||
assert_eq!(buckets[0].name.as_deref(), Some("bucket-a"));
|
||||
assert_eq!(buckets[0].creation_date, Some(s3s::dto::Timestamp::from(OffsetDateTime::UNIX_EPOCH)));
|
||||
assert_eq!(buckets[1].name.as_deref(), Some("bucket-b"));
|
||||
assert_eq!(buckets[1].creation_date, Some(s3s::dto::Timestamp::from(OffsetDateTime::UNIX_EPOCH)));
|
||||
|
||||
let expected_owner = rustfs_owner();
|
||||
assert_eq!(owner.display_name, expected_owner.display_name);
|
||||
assert_eq!(owner.id, expected_owner.id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_objects_marker_echoes_request_value() {
|
||||
let output = build_list_objects_output(ListObjectsV2Output::default(), Some("m-1".to_string()));
|
||||
|
||||
@@ -146,6 +146,10 @@ pub(crate) use super::sse::{
|
||||
};
|
||||
|
||||
pub(crate) mod access_consumer {
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) use super::super::access::{
|
||||
AccessMarkers, VerifiedSignature, admit_operation, authorize_bucket_request, resolve_req_info,
|
||||
};
|
||||
pub(crate) use super::super::access::{
|
||||
AuthorizationRequest, BucketConfigMutationSnapshot, BucketGenerationGuard, CopySourceBucketGenerationGuard,
|
||||
MultipartStagingOperation, ObjectTagConditions, OdmReadGenerationGuard, PendingDeleteBucketGenerationGuard,
|
||||
@@ -373,16 +377,12 @@ pub(crate) mod heal_control_startup_consumer {
|
||||
pub(crate) mod s3_api_consumer {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use super::super::super::s3_api::bucket::{
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_buckets_output, build_list_object_versions_output,
|
||||
build_list_objects_output, build_list_objects_v2_output, encode_list_objects_v2_continuation_token,
|
||||
parse_list_object_versions_params, parse_list_objects_v2_params,
|
||||
ListObjectVersionsParams, ListObjectsV2Params, build_list_object_versions_output, build_list_objects_output,
|
||||
build_list_objects_v2_output, encode_list_objects_v2_continuation_token, parse_list_object_versions_params,
|
||||
parse_list_objects_v2_params,
|
||||
};
|
||||
|
||||
pub(crate) mod contract {
|
||||
pub(crate) mod bucket {
|
||||
pub(crate) use super::super::super::super::contract::bucket::BucketInfo;
|
||||
}
|
||||
|
||||
pub(crate) mod list {
|
||||
pub(crate) use super::super::super::super::contract::list::{ListObjectVersionsInfo, ListObjectsV2Info};
|
||||
}
|
||||
@@ -395,6 +395,7 @@ pub(crate) mod s3_api_consumer {
|
||||
|
||||
pub(crate) mod common {
|
||||
pub(crate) use super::super::super::s3_api::common::rustfs_owner;
|
||||
pub(crate) use super::super::super::s3_api::common::{RUSTFS_OWNER_DISPLAY_NAME, RUSTFS_OWNER_ID};
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
|
||||
@@ -232,6 +232,14 @@ pub(crate) mod server {
|
||||
pub(crate) use crate::storage::storage_api::request_context_consumer::RequestContext;
|
||||
}
|
||||
|
||||
/// The access-phase functions the gateway stack shares with the legacy access hooks.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) mod access {
|
||||
pub(crate) use crate::storage::storage_api::access_consumer::{
|
||||
AccessMarkers, VerifiedSignature, admit_operation, authorize_bucket_request, resolve_req_info,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod rpc {
|
||||
pub(crate) use crate::storage::storage_api::rpc_consumer::InternodeRpcService;
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Refusal-matrix rows of migration batch A2 (rustfs/backlog#2765). None is served yet.
|
||||
|
||||
pub(crate) const MATRIX: &[super::MatrixRow] = &[];
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Refusal-matrix rows of migration batch A1 (rustfs/backlog#2765). None is served yet.
|
||||
|
||||
pub(crate) const MATRIX: &[super::MatrixRow] = &[];
|
||||
@@ -0,0 +1,43 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Refusal-matrix rows of migration batch A4 and of ListBuckets (rustfs/backlog#2765).
|
||||
|
||||
use super::{MatrixRow, Outcome, outcome};
|
||||
use aws_sdk_s3::Client;
|
||||
use futures::future::BoxFuture;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
|
||||
pub(crate) const MATRIX: &[MatrixRow] = &[
|
||||
MatrixRow {
|
||||
operation: "ListBuckets",
|
||||
metric: S3Operation::ListBuckets,
|
||||
has_error_body: true,
|
||||
call: list_buckets,
|
||||
},
|
||||
MatrixRow {
|
||||
operation: "HeadBucket",
|
||||
metric: S3Operation::HeadBucket,
|
||||
has_error_body: false,
|
||||
call: head_bucket,
|
||||
},
|
||||
];
|
||||
|
||||
fn list_buckets<'a>(client: &'a Client, _bucket: &'a str) -> BoxFuture<'a, Outcome> {
|
||||
Box::pin(async move { outcome(client.list_buckets().send().await) })
|
||||
}
|
||||
|
||||
fn head_bucket<'a>(client: &'a Client, bucket: &'a str) -> BoxFuture<'a, Outcome> {
|
||||
Box::pin(async move { outcome(client.head_bucket().bucket(bucket).send().await) })
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Refusal-matrix rows of migration batch A3 (rustfs/backlog#2765). None is served yet.
|
||||
|
||||
pub(crate) const MATRIX: &[super::MatrixRow] = &[];
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Refusal-matrix rows of migration batch B2 (rustfs/backlog#2765). None is served yet.
|
||||
|
||||
pub(crate) const MATRIX: &[super::MatrixRow] = &[];
|
||||
@@ -0,0 +1,508 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The gateway S3 stack inside an embedded RustFS server (rustfs/backlog#2734). Requests are signed
|
||||
// by the AWS SDK and travel through the whole middleware stack, so what is asserted here is what a
|
||||
// client sees. Every served operation has a row in the refusal matrix, kept per migration family in
|
||||
// the module named after it, and the matrix runs on both stacks.
|
||||
|
||||
#![recursion_limit = "256"]
|
||||
|
||||
#[path = "../common/mod.rs"]
|
||||
mod common;
|
||||
|
||||
mod bucket_access;
|
||||
mod bucket_configs;
|
||||
mod bucket_core;
|
||||
mod bucket_replication_lifecycle;
|
||||
mod listing_multipart;
|
||||
mod object_commit;
|
||||
mod object_meta;
|
||||
mod object_misc;
|
||||
mod object_read;
|
||||
mod object_write;
|
||||
|
||||
use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use futures::future::BoxFuture;
|
||||
use rustfs::app::context::get_global_app_context;
|
||||
use rustfs::embedded::{RustFSServer, RustFSServerBuilder, find_available_port, gateway_operations};
|
||||
use rustfs_io_metrics::s3_http_metrics::s3_http_metrics_snapshot;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_storage_api::{BucketOperations, MakeBucketOptions};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::future::Future;
|
||||
|
||||
const ROOT_ACCESS_KEY: &str = "gatewaystackroot";
|
||||
const ROOT_SECRET_KEY: &str = "gateway-stack-root-secret";
|
||||
const USER_SECRET_KEY: &str = "gateway-stack-user-secret";
|
||||
const NO_POLICY_USER: &str = "gatewaystacknopolicy";
|
||||
const MATRIX_BUCKET: &str = "gateway-stack-matrix";
|
||||
const ENV_S3_STACK: &str = "RUSTFS_S3_STACK";
|
||||
const ENV_KEYSTONE_ENABLE: &str = "RUSTFS_KEYSTONE_ENABLE";
|
||||
const ENV_KEYSTONE_AUTH_URL: &str = "RUSTFS_KEYSTONE_AUTH_URL";
|
||||
/// A Keystone endpoint nothing listens on: these tests are about start-up, not about tokens.
|
||||
const UNREACHABLE_KEYSTONE: &str = "http://127.0.0.1:1/v3";
|
||||
const GATEWAY: Option<&str> = Some("gateway");
|
||||
const LEGACY: Option<&str> = None;
|
||||
|
||||
/// One served operation as a client calls it.
|
||||
pub(crate) struct MatrixRow {
|
||||
pub(crate) operation: &'static str,
|
||||
/// The label the S3 request counter counts a served call under.
|
||||
pub(crate) metric: S3Operation,
|
||||
/// Whether a refusal carries an error document. A HEAD answer carries none, so no code.
|
||||
pub(crate) has_error_body: bool,
|
||||
/// Calls the operation on the bucket named by the second argument.
|
||||
pub(crate) call: for<'a> fn(&'a Client, &'a str) -> BoxFuture<'a, Outcome>,
|
||||
}
|
||||
|
||||
/// Every family's rows, in the order of the migration batches. A batch appends to its own module.
|
||||
const MATRIX: &[&[MatrixRow]] = &[
|
||||
bucket_core::MATRIX,
|
||||
bucket_configs::MATRIX,
|
||||
bucket_access::MATRIX,
|
||||
bucket_replication_lifecycle::MATRIX,
|
||||
object_meta::MATRIX,
|
||||
listing_multipart::MATRIX,
|
||||
object_read::MATRIX,
|
||||
object_write::MATRIX,
|
||||
object_commit::MATRIX,
|
||||
object_misc::MATRIX,
|
||||
];
|
||||
|
||||
fn matrix() -> impl Iterator<Item = &'static MatrixRow> {
|
||||
MATRIX.iter().flat_map(|family| family.iter())
|
||||
}
|
||||
|
||||
/// What one call observed: served, or refused.
|
||||
pub(crate) type Outcome = Result<(), Refusal>;
|
||||
|
||||
/// A refusal as the client saw it: the error code, the status, and both request-id headers.
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub(crate) struct Refusal {
|
||||
code: Option<String>,
|
||||
status: Option<u16>,
|
||||
request_ids: [Option<String>; 2],
|
||||
}
|
||||
|
||||
pub(crate) fn outcome<T, E: ProvideErrorMetadata>(result: Result<T, SdkError<E>>) -> Outcome {
|
||||
let error = match result {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(error) => error,
|
||||
};
|
||||
let response = error.raw_response();
|
||||
let header = |name: &str| response.and_then(|response| response.headers().get(name)).map(str::to_owned);
|
||||
Err(Refusal {
|
||||
code: error.code().map(str::to_owned),
|
||||
status: response.map(|response| response.status().as_u16()),
|
||||
request_ids: [header("x-amz-request-id"), header("x-request-id")],
|
||||
})
|
||||
}
|
||||
|
||||
fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client {
|
||||
let config = Config::builder()
|
||||
.credentials_provider(Credentials::new(access_key, secret_key, None, None, "test"))
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(endpoint)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
}
|
||||
|
||||
/// A client that signs nothing.
|
||||
async fn anonymous_client(endpoint: &str) -> Client {
|
||||
let shared = aws_config::defaults(BehaviorVersion::latest())
|
||||
.no_credentials()
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(endpoint)
|
||||
.load()
|
||||
.await;
|
||||
Client::from_conf(aws_sdk_s3::config::Builder::from(&shared).force_path_style(true).build())
|
||||
}
|
||||
|
||||
/// Runs `test` against an embedded server started with `RUSTFS_S3_STACK` set to `stack`.
|
||||
fn with_stack<F, Fut>(stack: Option<&'static str>, test: F)
|
||||
where
|
||||
F: FnOnce(RustFSServer) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = ()> + 'static,
|
||||
{
|
||||
with_environment(vec![(ENV_S3_STACK, stack)], test);
|
||||
}
|
||||
|
||||
/// Runs `test` against an embedded server started under `environment`.
|
||||
fn with_environment<F, Fut>(environment: Vec<(&'static str, Option<&'static str>)>, test: F)
|
||||
where
|
||||
F: FnOnce(RustFSServer) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = ()> + 'static,
|
||||
{
|
||||
temp_env::with_vars(environment, || {
|
||||
common::run_embedded_test(move || async move {
|
||||
test(start_server().await.expect("start embedded server")).await;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/// Starts an embedded server on a free port. A host without a free port fails the test rather
|
||||
/// than passing it unobserved.
|
||||
async fn start_server() -> Result<RustFSServer, String> {
|
||||
let port = find_available_port().expect("a free local port");
|
||||
let started = RustFSServerBuilder::new()
|
||||
.address(format!("127.0.0.1:{port}"))
|
||||
.access_key(ROOT_ACCESS_KEY)
|
||||
.secret_key(ROOT_SECRET_KEY)
|
||||
.build()
|
||||
.await;
|
||||
// Both renderings: the refusal's text may be in either.
|
||||
started.map_err(|error| format!("{error} {error:?}"))
|
||||
}
|
||||
|
||||
/// The S3 error code and HTTP status an SDK call was refused with.
|
||||
fn refusal<E: ProvideErrorMetadata>(error: &SdkError<E>) -> (Option<String>, Option<u16>) {
|
||||
let status = error.raw_response().map(|response| response.status().as_u16());
|
||||
(error.code().map(str::to_owned), status)
|
||||
}
|
||||
|
||||
/// One series of the S3 request counter: method, operation and outcome.
|
||||
type Series = (&'static str, &'static str, &'static str);
|
||||
|
||||
/// How often each series of the S3 request counter has been counted in this process.
|
||||
fn s3_request_counts() -> BTreeMap<Series, u64> {
|
||||
s3_http_metrics_snapshot()
|
||||
.into_iter()
|
||||
.map(|series| ((series.method, series.operation, series.outcome), series.total))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Every series counted since `before`, with how often. Reading the whole counter, not one
|
||||
/// series, is what makes a second count under another label visible.
|
||||
fn counted_since(before: &BTreeMap<Series, u64>) -> BTreeMap<Series, u64> {
|
||||
s3_request_counts()
|
||||
.into_iter()
|
||||
.filter_map(|(series, total)| {
|
||||
let counted = total - before.get(&series).copied().unwrap_or(0);
|
||||
(counted > 0).then_some((series, counted))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn create_bucket_in_the_store(bucket: &str) {
|
||||
get_global_app_context()
|
||||
.expect("embedded server context")
|
||||
.object_store()
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket in the store");
|
||||
}
|
||||
|
||||
async fn create_user_without_a_policy(access_key: &str) {
|
||||
get_global_app_context()
|
||||
.expect("embedded server context")
|
||||
.iam()
|
||||
.handle()
|
||||
.create_user(
|
||||
access_key,
|
||||
&rustfs_madmin::AddOrUpdateUserReq {
|
||||
secret_key: USER_SECRET_KEY.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create user without a policy");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_served_operation_has_one_matrix_row_and_every_row_a_served_operation() {
|
||||
let rows: Vec<&str> = matrix().map(|row| row.operation).collect();
|
||||
let distinct: BTreeSet<&str> = rows.iter().copied().collect();
|
||||
|
||||
assert_eq!(rows.len(), distinct.len(), "an operation has two rows: {rows:?}");
|
||||
assert_eq!(distinct, gateway_operations().into_iter().collect());
|
||||
}
|
||||
|
||||
/// Every served operation serves the root caller, counted once under its label, and refuses
|
||||
/// each caller it cannot admit with the code the legacy stack answers, counted once.
|
||||
async fn every_served_operation_serves_root_and_refuses_the_callers_it_cannot_admit(server: RustFSServer) {
|
||||
create_bucket_in_the_store(MATRIX_BUCKET).await;
|
||||
create_user_without_a_policy(NO_POLICY_USER).await;
|
||||
let endpoint = server.endpoint();
|
||||
let root = s3_client(&endpoint, server.access_key(), server.secret_key());
|
||||
// The caller, its client, the code it is refused with, and whether authentication refuses it
|
||||
// before any operation is known.
|
||||
let refused = [
|
||||
("anonymous", anonymous_client(&endpoint).await, "AccessDenied", false),
|
||||
(
|
||||
"wrong secret",
|
||||
s3_client(&endpoint, ROOT_ACCESS_KEY, "not-the-root-secret"),
|
||||
"SignatureDoesNotMatch",
|
||||
true,
|
||||
),
|
||||
(
|
||||
"unknown key",
|
||||
s3_client(&endpoint, "gatewaystackunknown", USER_SECRET_KEY),
|
||||
"InvalidAccessKeyId",
|
||||
true,
|
||||
),
|
||||
("no policy", s3_client(&endpoint, NO_POLICY_USER, USER_SECRET_KEY), "AccessDenied", false),
|
||||
];
|
||||
|
||||
// A caller served where it must be refused is collected, so one run names every such caller.
|
||||
let mut served = Vec::new();
|
||||
for row in matrix() {
|
||||
let before = s3_request_counts();
|
||||
assert_eq!((row.call)(&root, MATRIX_BUCKET).await, Ok(()), "{} as root", row.operation);
|
||||
let counted = counted_since(&before);
|
||||
assert_eq!(counted.len(), 1, "{}: {counted:?}", row.operation);
|
||||
assert!(
|
||||
counted
|
||||
.iter()
|
||||
.all(|((_, label, outcome), total)| (*label, *outcome, *total) == (row.metric.as_str(), "2xx", 1)),
|
||||
"{}: {counted:?}",
|
||||
row.operation
|
||||
);
|
||||
|
||||
for (caller, client, code, by_authentication) in &refused {
|
||||
let before = s3_request_counts();
|
||||
let refusal = match (row.call)(client, MATRIX_BUCKET).await {
|
||||
Ok(()) => {
|
||||
served.push(format!("{} serves the {caller} caller", row.operation));
|
||||
continue;
|
||||
}
|
||||
Err(refusal) => refusal,
|
||||
};
|
||||
let context = format!("{} as the {caller} caller: {refusal:?}", row.operation);
|
||||
assert_eq!(refusal.status, Some(403), "{context}");
|
||||
if row.has_error_body {
|
||||
assert_eq!(refusal.code.as_deref(), Some(*code), "{context}");
|
||||
}
|
||||
assert!(refusal.request_ids[0].is_some(), "the refusal names its request: {context}");
|
||||
assert_eq!(
|
||||
refusal.request_ids[0], refusal.request_ids[1],
|
||||
"one identifier in both headers: {context}"
|
||||
);
|
||||
let counted = counted_since(&before);
|
||||
assert_eq!(counted.values().sum::<u64>(), 1, "counted once: {context} {counted:?}");
|
||||
assert!(counted.keys().all(|(_, _, outcome)| *outcome == "4xx"), "{context} {counted:?}");
|
||||
if *by_authentication {
|
||||
assert!(
|
||||
counted.keys().all(|(_, label, _)| *label == "unknown"),
|
||||
"under no operation: {context} {counted:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(served.is_empty(), "{served:?}");
|
||||
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_refusal_matrix_holds_on_the_gateway_stack() {
|
||||
with_stack(GATEWAY, every_served_operation_serves_root_and_refuses_the_callers_it_cannot_admit);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_refusal_matrix_holds_on_the_legacy_stack() {
|
||||
with_stack(LEGACY, every_served_operation_serves_root_and_refuses_the_callers_it_cannot_admit);
|
||||
}
|
||||
|
||||
/// Names RustFS creates and the AWS bucket grammar refuses. An IP-shaped name such as `256.1.1.1`
|
||||
/// is also outside the AWS grammar, but RustFS's own create rule refuses it, so no store holds one.
|
||||
const RUSTFS_ONLY_NAMES: [&str; 5] = ["a-s3alias", "sthree-x", "x--ol-s3", "x--x-s3", "xn--abc"];
|
||||
/// HeadBucket statuses the legacy stack answers on `main` for names no store holds.
|
||||
const HEAD_UPPER: u16 = 400;
|
||||
const HEAD_DOTTED_QUAD: u16 = 404;
|
||||
const HEAD_TOO_SHORT: u16 = 400;
|
||||
const HEAD_TOO_LONG: u16 = 400;
|
||||
|
||||
/// A listing names every bucket the store holds, as it is stored, with its date and the owner.
|
||||
async fn the_listing_names_every_stored_bucket(server: RustFSServer) {
|
||||
for name in RUSTFS_ONLY_NAMES {
|
||||
create_bucket_in_the_store(name).await;
|
||||
}
|
||||
|
||||
let listing = s3_client(&server.endpoint(), server.access_key(), server.secret_key())
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect("signed ListBuckets");
|
||||
|
||||
let mut names: Vec<&str> = listing.buckets().iter().filter_map(|bucket| bucket.name()).collect();
|
||||
names.sort_unstable();
|
||||
let mut expected = RUSTFS_ONLY_NAMES;
|
||||
expected.sort_unstable();
|
||||
assert_eq!(names, expected);
|
||||
assert!(listing.owner().and_then(|owner| owner.id()).is_some(), "the listing names its owner");
|
||||
assert!(
|
||||
listing.buckets().iter().all(|bucket| bucket.creation_date().is_some()),
|
||||
"every bucket carries its creation date"
|
||||
);
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_lists_every_stored_bucket() {
|
||||
with_stack(GATEWAY, the_listing_names_every_stored_bucket);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_stack_lists_every_stored_bucket() {
|
||||
with_stack(LEGACY, the_listing_names_every_stored_bucket);
|
||||
}
|
||||
|
||||
/// HeadBucket on the legacy stack answers every name as before the use case moved onto the
|
||||
/// gateway's types: a stored name the AWS grammar refuses is found, a missing one is not, and a
|
||||
/// name the edge refuses is refused there. Never a server fault. The statuses were measured on
|
||||
/// `main` before this change.
|
||||
#[test]
|
||||
fn the_legacy_stack_heads_every_bucket_name_as_before() {
|
||||
with_stack(LEGACY, |server| async move {
|
||||
for name in RUSTFS_ONLY_NAMES {
|
||||
create_bucket_in_the_store(name).await;
|
||||
}
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
let too_long = "a".repeat(64);
|
||||
let expected = RUSTFS_ONLY_NAMES.iter().map(|name| (*name, 200)).chain([
|
||||
("missing-bucket", 404),
|
||||
("UPPER", HEAD_UPPER),
|
||||
("256.1.1.1", HEAD_DOTTED_QUAD),
|
||||
("ab", HEAD_TOO_SHORT),
|
||||
(too_long.as_str(), HEAD_TOO_LONG),
|
||||
]);
|
||||
|
||||
let mut answered = Vec::new();
|
||||
let mut wanted = Vec::new();
|
||||
for (name, status) in expected {
|
||||
let observed = match client.head_bucket().bucket(name).send().await {
|
||||
Ok(_) => Some(200),
|
||||
Err(error) => refusal(&error).1,
|
||||
};
|
||||
answered.push((name.to_owned(), observed));
|
||||
wanted.push((name.to_owned(), Some(status)));
|
||||
}
|
||||
assert_eq!(answered, wanted);
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
/// The PutObject-only upload-size capability is refused on any other operation, before it is
|
||||
/// served, with the same answer on both stacks. The legacy stack refuses it in the admission both
|
||||
/// edges share; the gateway refuses the query before its handler runs.
|
||||
async fn a_misplaced_capability_query_is_refused(server: RustFSServer) {
|
||||
let refused = s3_client(&server.endpoint(), server.access_key(), server.secret_key())
|
||||
.list_buckets()
|
||||
.customize()
|
||||
.mutate_request(|request| {
|
||||
let target = format!("{}?x-rustfs-max-content-length=1", request.uri());
|
||||
request.set_uri(target).expect("a valid target");
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a capability query is refused on ListBuckets");
|
||||
|
||||
assert_eq!(refusal(&refused), (Some("InvalidRequest".to_owned()), Some(400)));
|
||||
server.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gateway_stack_refuses_a_misplaced_capability_query() {
|
||||
with_stack(GATEWAY, a_misplaced_capability_query_is_refused);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_stack_refuses_a_misplaced_capability_query() {
|
||||
with_stack(LEGACY, a_misplaced_capability_query_is_refused);
|
||||
}
|
||||
|
||||
/// The pinned gateway admits a request that carries only `X-Auth-Token` as anonymous, while the
|
||||
/// RustFS code behind it reads the Keystone identity. Until the gateway authenticates the token
|
||||
/// (rustfs/gateway#1586) the two settings together refuse to start, and nothing is served.
|
||||
#[test]
|
||||
fn the_gateway_stack_refuses_to_start_with_keystone_enabled() {
|
||||
let environment = [
|
||||
(ENV_S3_STACK, GATEWAY),
|
||||
(ENV_KEYSTONE_ENABLE, Some("true")),
|
||||
(ENV_KEYSTONE_AUTH_URL, Some(UNREACHABLE_KEYSTONE)),
|
||||
];
|
||||
temp_env::with_vars(environment, || {
|
||||
common::run_embedded_test(|| async {
|
||||
let refusal = match start_server().await {
|
||||
Ok(server) => {
|
||||
server.shutdown().await;
|
||||
panic!("the gateway stack started with Keystone enabled");
|
||||
}
|
||||
Err(refusal) => refusal,
|
||||
};
|
||||
for named in ["RUSTFS_S3_STACK=gateway", "RUSTFS_KEYSTONE_ENABLE", "rustfs/gateway#1586"] {
|
||||
assert!(refusal.contains(named), "{named} is missing from: {refusal}");
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_legacy_stack_starts_with_keystone_enabled() {
|
||||
let environment = vec![
|
||||
(ENV_S3_STACK, LEGACY),
|
||||
(ENV_KEYSTONE_ENABLE, Some("true")),
|
||||
(ENV_KEYSTONE_AUTH_URL, Some(UNREACHABLE_KEYSTONE)),
|
||||
];
|
||||
with_environment(environment, |server| async move {
|
||||
let listing = s3_client(&server.endpoint(), server.access_key(), server.secret_key())
|
||||
.list_buckets()
|
||||
.send()
|
||||
.await
|
||||
.expect("signed ListBuckets on the legacy stack with Keystone enabled");
|
||||
assert!(listing.buckets().is_empty());
|
||||
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
|
||||
/// The legacy stack still serves operations the gateway stack does not, in the same binary.
|
||||
#[test]
|
||||
fn the_legacy_stack_still_serves_writes_and_reads() {
|
||||
with_stack(LEGACY, |server| async move {
|
||||
let client = s3_client(&server.endpoint(), server.access_key(), server.secret_key());
|
||||
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket("legacy-stack-alpha")
|
||||
.send()
|
||||
.await
|
||||
.expect("CreateBucket on the legacy stack");
|
||||
client
|
||||
.put_object()
|
||||
.bucket("legacy-stack-alpha")
|
||||
.key("object")
|
||||
.body(ByteStream::from_static(b"payload"))
|
||||
.send()
|
||||
.await
|
||||
.expect("PutObject on the legacy stack");
|
||||
let missing = client
|
||||
.get_object()
|
||||
.bucket("legacy-stack-alpha")
|
||||
.key("missing")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a missing key");
|
||||
assert_eq!(refusal(&missing), (Some("NoSuchKey".to_owned()), Some(404)));
|
||||
|
||||
server.shutdown().await;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Refusal-matrix rows of migration batch E1 (rustfs/backlog#2765). None is served yet.
|
||||
|
||||
pub(crate) const MATRIX: &[super::MatrixRow] = &[];
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Refusal-matrix rows of migration batch B1 (rustfs/backlog#2765). None is served yet.
|
||||
|
||||
pub(crate) const MATRIX: &[super::MatrixRow] = &[];
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Refusal-matrix rows of migration batch E2 (rustfs/backlog#2765). None is served yet.
|
||||
|
||||
pub(crate) const MATRIX: &[super::MatrixRow] = &[];
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Refusal-matrix rows of migration batch C (rustfs/backlog#2765). None is served yet.
|
||||
|
||||
pub(crate) const MATRIX: &[super::MatrixRow] = &[];
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Refusal-matrix rows of migration batch D (rustfs/backlog#2765). None is served yet.
|
||||
|
||||
pub(crate) const MATRIX: &[super::MatrixRow] = &[];
|
||||
@@ -1,18 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove the normal release compiles and the debug-only feature is rejected."""
|
||||
"""Keep every debug-only feature of the rustfs crate out of release builds.
|
||||
|
||||
Two observations per feature. Its feature closure, read from `cargo metadata`, is reached from no
|
||||
feature but the ones allowed to enable it (`--features-only` runs this alone). And a release build
|
||||
that enables it fails at its `compile_error!` guard, after the normal release configuration compiled.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
MESSAGE = "connect-e2e-short-credentials is restricted to debug builds"
|
||||
SOURCE = "rustfs/src/connect/registration.rs"
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Boundary:
|
||||
feature: str
|
||||
message: str
|
||||
source: str
|
||||
# Debug-only features allowed to enable it; none of them may be reachable from a release build.
|
||||
enablers: tuple[str, ...]
|
||||
|
||||
|
||||
def validate_rejection(status: int, output: str, root: Path) -> list[str]:
|
||||
BOUNDARIES = (
|
||||
Boundary("connect-e2e-short-credentials", "connect-e2e-short-credentials is restricted to debug builds",
|
||||
"rustfs/src/connect/registration.rs", ()),
|
||||
Boundary("gateway-stack", "gateway-stack is restricted to debug builds",
|
||||
"rustfs/src/server/gateway_stack.rs", ("e2e-test-hooks",)),
|
||||
)
|
||||
|
||||
|
||||
def package_features(root: Path) -> dict[str, list[str]]:
|
||||
command = ["cargo", "metadata", "--locked", "--no-deps", "--format-version", "1"]
|
||||
metadata = json.loads(subprocess.run(command, cwd=root, check=True, capture_output=True, text=True).stdout)
|
||||
return next(package["features"] for package in metadata["packages"] if package["name"] == "rustfs")
|
||||
|
||||
|
||||
def enables(features: dict[str, list[str]], start: str, feature: str) -> bool:
|
||||
seen, pending = set(), [start]
|
||||
while pending:
|
||||
for member in features.get(pending.pop(), []):
|
||||
if member == feature:
|
||||
return True
|
||||
if member in features and member not in seen:
|
||||
seen.add(member)
|
||||
pending.append(member)
|
||||
return False
|
||||
|
||||
|
||||
def closure_errors(features: dict[str, list[str]], boundary: Boundary) -> list[str]:
|
||||
if boundary.feature not in features:
|
||||
return [f"the {boundary.feature} feature this guard protects is gone"]
|
||||
errors = []
|
||||
for name in sorted(features):
|
||||
if name != boundary.feature and enables(features, name, boundary.feature) and name not in boundary.enablers:
|
||||
errors.append(f"feature {name} enables {boundary.feature}")
|
||||
for name in boundary.enablers:
|
||||
if name not in features or not enables(features, name, boundary.feature):
|
||||
errors.append(f"allowed enabler {name} no longer enables {boundary.feature}; update the guard")
|
||||
return errors
|
||||
|
||||
|
||||
def validate_rejection(status: int, output: str, root: Path, boundary: Boundary) -> list[str]:
|
||||
errors, diagnostics, completed = [], [], []
|
||||
for line in output.splitlines():
|
||||
try:
|
||||
@@ -34,7 +85,7 @@ def validate_rejection(status: int, output: str, root: Path) -> list[str]:
|
||||
return errors
|
||||
diagnostic = diagnostics[0]
|
||||
message = diagnostic["message"]
|
||||
if message.get("message") != MESSAGE or message.get("code") is not None:
|
||||
if message.get("message") != boundary.message or message.get("code") is not None:
|
||||
errors.append("release failed for an unrelated compiler error")
|
||||
target = diagnostic.get("target", {})
|
||||
if target.get("name") != "rustfs" or "lib" not in target.get("kind", []):
|
||||
@@ -44,44 +95,65 @@ def validate_rejection(status: int, output: str, root: Path) -> list[str]:
|
||||
for span in primary:
|
||||
path = Path(span.get("file_name", ""))
|
||||
path = (root / path).resolve() if not path.is_absolute() else path.resolve()
|
||||
if path != (root / SOURCE).resolve():
|
||||
if path != (root / boundary.source).resolve():
|
||||
continue
|
||||
try:
|
||||
source = path.read_text().splitlines()
|
||||
start, end = span["line_start"], span["line_end"]
|
||||
if type(start) is int and type(end) is int and 1 <= start <= end <= len(source):
|
||||
excerpt = "\n".join(source[start - 1:end])
|
||||
valid = "compile_error!" in source[start - 1] and MESSAGE in excerpt
|
||||
valid = "compile_error!" in source[start - 1] and boundary.message in excerpt
|
||||
except (OSError, KeyError, TypeError):
|
||||
pass
|
||||
if not valid:
|
||||
errors.append("release rejection is not located at the credential lifetime guard")
|
||||
errors.append(f"release rejection is not located at the guard in {boundary.source}")
|
||||
return errors
|
||||
|
||||
|
||||
def check(root: Path) -> int:
|
||||
def report(errors: list[str]) -> int:
|
||||
for error in errors:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return bool(errors)
|
||||
|
||||
|
||||
def check_features(root: Path, boundaries=BOUNDARIES) -> int:
|
||||
features = package_features(root)
|
||||
errors = [error for boundary in boundaries for error in closure_errors(features, boundary)]
|
||||
if not errors:
|
||||
print("Release feature boundary passed: no release-reachable feature enables a debug-only one")
|
||||
return report(errors)
|
||||
|
||||
|
||||
def check(root: Path, boundaries=BOUNDARIES) -> int:
|
||||
if check_features(root, boundaries):
|
||||
return 1
|
||||
command = ["cargo", "check", "--locked", "-p", "rustfs", "--lib", "--release", "--message-format=json"]
|
||||
positive = subprocess.run(command, cwd=root)
|
||||
if positive.returncode:
|
||||
print("ERROR: normal release configuration must compile before the negative check", file=sys.stderr)
|
||||
return 1
|
||||
negative = subprocess.run(command + ["--features", "connect-e2e-short-credentials"], cwd=root, capture_output=True, text=True)
|
||||
sys.stderr.write(negative.stderr)
|
||||
errors = validate_rejection(negative.returncode, negative.stdout, root)
|
||||
for error in errors:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
if errors:
|
||||
# Cargo JSON may contain useful source diagnostics; retain it on failure.
|
||||
sys.stdout.write(negative.stdout)
|
||||
else:
|
||||
print("Connect release boundary passed: normal configuration compiles; short credentials are rejected at their guard")
|
||||
return bool(errors)
|
||||
failed = False
|
||||
for boundary in boundaries:
|
||||
negative = subprocess.run(command + ["--features", boundary.feature], cwd=root, capture_output=True, text=True)
|
||||
sys.stderr.write(negative.stderr)
|
||||
errors = validate_rejection(negative.returncode, negative.stdout, root, boundary)
|
||||
if errors:
|
||||
# Cargo JSON may contain useful source diagnostics; retain it on failure.
|
||||
sys.stdout.write(negative.stdout)
|
||||
failed = True
|
||||
else:
|
||||
print(f"Release boundary passed: {boundary.feature} is rejected at its guard")
|
||||
report(errors)
|
||||
return failed
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if sys.argv[1:] == ["--self-test"]:
|
||||
suite = unittest.defaultTestLoader.discover(str(Path(__file__).resolve().parent), pattern="test_connect_release_boundary.py")
|
||||
raise SystemExit(not unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful())
|
||||
root = Path(__file__).resolve().parent.parent
|
||||
if sys.argv[1:] == ["--features-only"]:
|
||||
raise SystemExit(check_features(root))
|
||||
if sys.argv[1:]:
|
||||
raise SystemExit("usage: check_connect_release_boundary.py [--self-test]")
|
||||
raise SystemExit(check(Path(__file__).resolve().parent.parent))
|
||||
raise SystemExit("usage: check_connect_release_boundary.py [--self-test | --features-only]")
|
||||
raise SystemExit(check(root))
|
||||
|
||||
+4
-2
@@ -18,7 +18,7 @@ ROOT = Path(__file__).resolve().parent.parent
|
||||
ALWAYS_JOBS = ("classify-changes", "typos", "quick-checks")
|
||||
CODE_JOBS = (
|
||||
"workspace-clippy", "workspace-lint-and-build", "workspace-tests", "core-contract", "connect-feature-boundaries", "fips-provider", "test-and-lint-rio-v2",
|
||||
"test-and-lint-protocols", "build-rustfs-debug-binary", "uring-integration",
|
||||
"test-and-lint-protocols", "test-and-lint-gateway-stack", "build-rustfs-debug-binary", "uring-integration",
|
||||
"e2e-tests", "s3-implemented-tests", "s3-lifecycle-behavior-tests",
|
||||
)
|
||||
# Specialized boundary, hardware and alternate-feature lanes do not gate a
|
||||
@@ -34,7 +34,7 @@ NIGHTLY_JOBS = ("connect-feature-boundaries", "fips-provider", "uring-integratio
|
||||
# against this revision's expected job contract. The workflow refuses a plan
|
||||
# that does not carry the current marker and falls back to the full matrix, so
|
||||
# a base planner predating the contract cannot authorize a narrower selection.
|
||||
PLAN_CONTRACT = "7"
|
||||
PLAN_CONTRACT = "8"
|
||||
# These jobs still exercise the server and its black-box harness when only E2E
|
||||
# Rust sources change. Production code, manifests and shared test configuration
|
||||
# always select the full matrix. No production package depends on e2e_test.
|
||||
@@ -371,6 +371,8 @@ def configurations(jobs: dict[str, str]) -> list[dict]:
|
||||
("protocol-sftp-tests", "test-and-lint-protocols", "test", ["sftp"]),
|
||||
("protocol-http3-lint", "test-and-lint-protocols", "dev", ["http3", "swift"]),
|
||||
("protocol-http3-tests", "test-and-lint-protocols", "test", ["http3", "swift"]),
|
||||
("gateway-stack-lint", "test-and-lint-gateway-stack", "dev", ["gateway-stack"]),
|
||||
("gateway-stack-tests", "test-and-lint-gateway-stack", "test", ["gateway-stack"]),
|
||||
("server-rio-v2", "build-rustfs-debug-binary-rio-v2", "dev", ["e2e-test-hooks", "rio-v2"]),
|
||||
]
|
||||
return [{"id": identity, "job": job, "target": "x86_64-unknown-linux-gnu", "profile": profile,
|
||||
|
||||
@@ -9,73 +9,115 @@ from unittest.mock import patch
|
||||
|
||||
import check_connect_release_boundary as guard
|
||||
|
||||
CONNECT, GATEWAY = guard.BOUNDARIES
|
||||
TOOLCHAIN = Path(__file__).resolve().parents[1] / 'rust-toolchain.toml'
|
||||
LOCK = 'version = 3\n\n[[package]]\nname = "rustfs"\nversion = "0.1.0"\n'
|
||||
|
||||
|
||||
def write_package(root: Path, features: str, source: str = guard.BOUNDARIES[0].source) -> None:
|
||||
(root / 'Cargo.toml').write_text(
|
||||
'[package]\nname = "rustfs"\nversion = "0.1.0"\nedition = "2021"\n'
|
||||
'[lib]\npath = "' + source + '"\n'
|
||||
'[features]\n' + features + '\n[workspace]\n'
|
||||
)
|
||||
(root / 'Cargo.lock').write_text(LOCK)
|
||||
# The fixture is outside the checkout, so retain its selected toolchain.
|
||||
(root / TOOLCHAIN.name).write_text(TOOLCHAIN.read_text())
|
||||
|
||||
|
||||
class ReleaseBoundaryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temp.name)
|
||||
source = self.root / guard.SOURCE
|
||||
source = self.root / CONNECT.source
|
||||
source.parent.mkdir(parents=True)
|
||||
source.write_text('// header\ncompile_error!("' + guard.MESSAGE + '");\n')
|
||||
source.write_text('// header\ncompile_error!("' + CONNECT.message + '");\n')
|
||||
self.diagnostic = {"reason": "compiler-message", "target": {"name": "rustfs", "kind": ["lib"]},
|
||||
"message": {"level": "error", "message": guard.MESSAGE, "code": None,
|
||||
"spans": [{"file_name": guard.SOURCE, "is_primary": True, "line_start": 2, "line_end": 2}]}}
|
||||
"message": {"level": "error", "message": CONNECT.message, "code": None,
|
||||
"spans": [{"file_name": CONNECT.source, "is_primary": True, "line_start": 2, "line_end": 2}]}}
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
|
||||
def output(self, *diagnostics):
|
||||
return "\n".join(json.dumps(value) for value in [*diagnostics, {"reason": "build-finished", "success": False}])
|
||||
|
||||
def test_rejection_is_bound_to_diagnostic_kind_and_source_guard(self):
|
||||
self.assertEqual(guard.validate_rejection(101, self.output(self.diagnostic), self.root), [])
|
||||
self.assertEqual(guard.validate_rejection(101, self.output(self.diagnostic), self.root, CONNECT), [])
|
||||
for status in (0, 1, 137):
|
||||
self.assertTrue(guard.validate_rejection(status, self.output(self.diagnostic), self.root))
|
||||
self.assertTrue(guard.validate_rejection(status, self.output(self.diagnostic), self.root, CONNECT))
|
||||
for field, value in (("message", "unrelated compiler error"), ("code", {"code": "E0599"}), ("level", "warning")):
|
||||
diagnostic = copy.deepcopy(self.diagnostic)
|
||||
diagnostic["message"][field] = value
|
||||
self.assertTrue(guard.validate_rejection(101, self.output(diagnostic), self.root))
|
||||
self.assertTrue(guard.validate_rejection(101, self.output(diagnostic), self.root, CONNECT))
|
||||
for field, value in (("file_name", "rustfs/src/main.rs"), ("is_primary", False), ("line_start", 1), ("line_start", 0), ("line_end", 100)):
|
||||
diagnostic = copy.deepcopy(self.diagnostic)
|
||||
diagnostic["message"]["spans"][0][field] = value
|
||||
self.assertTrue(guard.validate_rejection(101, self.output(diagnostic), self.root), (field, value))
|
||||
self.assertTrue(guard.validate_rejection(101, self.output(diagnostic), self.root, CONNECT), (field, value))
|
||||
|
||||
def test_a_rejection_proves_only_its_own_boundary(self):
|
||||
self.assertTrue(guard.validate_rejection(101, self.output(self.diagnostic), self.root, GATEWAY))
|
||||
|
||||
def test_other_errors_or_incomplete_cargo_output_cannot_prove_boundary(self):
|
||||
self.assertTrue(guard.validate_rejection(101, self.output(), self.root))
|
||||
self.assertTrue(guard.validate_rejection(101, self.output(self.diagnostic, self.diagnostic), self.root))
|
||||
self.assertTrue(guard.validate_rejection(101, json.dumps(self.diagnostic), self.root))
|
||||
self.assertTrue(guard.validate_rejection(101, self.output(self.diagnostic) + '\nerror: network failure', self.root))
|
||||
(self.root / guard.SOURCE).write_text('fn removed_guard() {}\n')
|
||||
self.assertTrue(guard.validate_rejection(101, self.output(self.diagnostic), self.root))
|
||||
self.assertTrue(guard.validate_rejection(101, self.output(), self.root, CONNECT))
|
||||
self.assertTrue(guard.validate_rejection(101, self.output(self.diagnostic, self.diagnostic), self.root, CONNECT))
|
||||
self.assertTrue(guard.validate_rejection(101, json.dumps(self.diagnostic), self.root, CONNECT))
|
||||
self.assertTrue(guard.validate_rejection(101, self.output(self.diagnostic) + '\nerror: network failure', self.root, CONNECT))
|
||||
(self.root / CONNECT.source).write_text('fn removed_guard() {}\n')
|
||||
self.assertTrue(guard.validate_rejection(101, self.output(self.diagnostic), self.root, CONNECT))
|
||||
|
||||
def test_a_feature_closure_holds_only_with_its_allowed_enablers(self):
|
||||
valid = {"default": ["ftps"], "full": ["ftps"], "ftps": [], "e2e-test-hooks": ["dep:x", "gateway-stack"],
|
||||
"gateway-stack": ["dep:rustfs-gateway"]}
|
||||
self.assertEqual(guard.closure_errors(valid, GATEWAY), [])
|
||||
self.assertTrue(guard.closure_errors({**valid, "full": ["ftps", "gateway-stack"]}, GATEWAY))
|
||||
self.assertTrue(guard.closure_errors({**valid, "default": ["hooks"], "hooks": ["e2e-test-hooks"]}, GATEWAY))
|
||||
self.assertTrue(guard.closure_errors({key: value for key, value in valid.items() if key != "gateway-stack"}, GATEWAY))
|
||||
self.assertTrue(guard.closure_errors({**valid, "e2e-test-hooks": ["dep:x"]}, GATEWAY))
|
||||
self.assertEqual(guard.closure_errors({"default": [], CONNECT.feature: []}, CONNECT), [])
|
||||
|
||||
def test_normal_configuration_failure_stops_negative_build(self):
|
||||
with patch.object(guard.subprocess, 'run') as run:
|
||||
with patch.object(guard, 'check_features', return_value=0), patch.object(guard.subprocess, 'run') as run:
|
||||
run.return_value.returncode = 101
|
||||
self.assertEqual(guard.check(self.root), 1)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
|
||||
def test_a_closure_failure_stops_every_build(self):
|
||||
with patch.object(guard, 'check_features', return_value=1), patch.object(guard.subprocess, 'run') as run:
|
||||
self.assertEqual(guard.check(self.root), 1)
|
||||
run.assert_not_called()
|
||||
|
||||
def test_real_cargo_metadata_sees_features_enabled_through_multi_line_arrays(self):
|
||||
# Each manifest once let a line-based reading of the feature table call the boundary held.
|
||||
fixtures = {
|
||||
'full lists the feature across lines': 'default = ["ftps"]\nftps = []\nfull = [\n "ftps",\n "gateway-stack",\n]\n'
|
||||
'e2e-test-hooks = ["gateway-stack"]\ngateway-stack = []\n',
|
||||
'full enables it through a multi-line feature': 'default = ["ftps"]\nftps = []\nfull = ["ftps", "hooks"]\n'
|
||||
'hooks = [\n "gateway-stack",\n]\ne2e-test-hooks = ["gateway-stack"]\ngateway-stack = []\n',
|
||||
'a release script can name a feature that enables it': 'default = ["ftps"]\nftps = []\nfull = ["ftps"]\n'
|
||||
'hooks = [\n "gateway-stack",\n]\ne2e-test-hooks = ["gateway-stack"]\ngateway-stack = []\n',
|
||||
}
|
||||
valid = 'default = ["ftps"]\nftps = []\nfull = ["ftps"]\ne2e-test-hooks = [\n "gateway-stack",\n]\ngateway-stack = []\n'
|
||||
with patch.dict(os.environ, {'CARGO_NET_OFFLINE': 'true'}):
|
||||
write_package(self.root, valid)
|
||||
self.assertEqual(guard.check_features(self.root, (GATEWAY,)), 0)
|
||||
for name, features in fixtures.items():
|
||||
with self.subTest(name=name):
|
||||
write_package(self.root, features)
|
||||
self.assertEqual(guard.check_features(self.root, (GATEWAY,)), 1)
|
||||
|
||||
def test_real_cargo_release_boundary(self):
|
||||
(self.root / 'Cargo.toml').write_text(
|
||||
'[package]\nname = "rustfs"\nversion = "0.1.0"\nedition = "2021"\n'
|
||||
'[lib]\npath = "' + guard.SOURCE + '"\n'
|
||||
'[features]\nconnect-e2e-short-credentials = []\n[workspace]\n'
|
||||
)
|
||||
(self.root / 'Cargo.lock').write_text(
|
||||
'version = 3\n\n[[package]]\nname = "rustfs"\nversion = "0.1.0"\n'
|
||||
)
|
||||
# The fixture is outside the checkout, so retain its selected toolchain
|
||||
# while keeping compilation offline and out of the workspace target.
|
||||
toolchain = Path(__file__).resolve().parents[1] / 'rust-toolchain.toml'
|
||||
(self.root / toolchain.name).write_text(toolchain.read_text())
|
||||
condition = '#[cfg(all(feature = "connect-e2e-short-credentials", not(debug_assertions)))]\n'
|
||||
write_package(self.root, CONNECT.feature + ' = []')
|
||||
condition = '#[cfg(all(feature = "' + CONNECT.feature + '", not(debug_assertions)))]\n'
|
||||
cases = (
|
||||
('expected rejection', condition + 'compile_error!("' + guard.MESSAGE + '");\n', 0),
|
||||
('expected rejection', condition + 'compile_error!("' + CONNECT.message + '");\n', 0),
|
||||
('missing rejection', '', 1),
|
||||
('unrelated rejection', condition + 'compile_error!("unrelated compiler error");\n', 1),
|
||||
)
|
||||
# Compilation stays offline and out of the workspace target.
|
||||
with patch.dict(os.environ, {'CARGO_TARGET_DIR': str(self.root / 'target'), 'CARGO_NET_OFFLINE': 'true'}):
|
||||
for name, source, expected in cases:
|
||||
with self.subTest(name=name):
|
||||
(self.root / guard.SOURCE).write_text(source)
|
||||
self.assertEqual(guard.check(self.root), expected)
|
||||
(self.root / CONNECT.source).write_text(source)
|
||||
self.assertEqual(guard.check(self.root, (CONNECT,)), expected)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in New Issue
Block a user