Compare commits

..

2 Commits

Author SHA1 Message Date
唐小鸭 d69d682193 Refactor code structure for improved readability and maintainability 2026-07-24 14:01:40 +08:00
唐小鸭 5ea9a1fd8f fix(sse): separate SSE-S3 and KMS key providers 2026-07-22 21:05:23 +08:00
75 changed files with 5078 additions and 4679 deletions
+30 -8
View File
@@ -15,18 +15,37 @@
# MinIO on-disk interop: prove RustFS reads MinIO-written erasure-coded SSE
# objects with byte-identical data and correct logical size.
#
# This is NOT a PR gate. The fixtures are real MinIO backend trees generated on
# the fly (they are gitignored, never committed), so the job regenerates them
# each run with Docker and then runs the `#[ignore]` reader tests in
# crates/ecstore/tests/minio_generated_read_test.rs.
# The fixtures are real MinIO backend trees generated on the fly (they are
# gitignored, never committed), so the job regenerates them each run with
# Docker and then runs the `#[ignore]` reader tests in
# crates/ecstore/tests/minio_generated_read_test.rs. SSE/RIO changes run this
# as a path-filtered PR gate; nightly and manual runs retain broader coverage.
#
# Runner: GitHub-hosted `ubuntu-latest`. It reliably ships Docker + Python,
# unlike the self-hosted fleet, whose pods drift in Docker/pip availability
# (see the infra note in e2e-s3tests.yml). Nightly + manual only.
# (see the infra note in e2e-s3tests.yml).
name: minio-interop
on:
workflow_dispatch:
pull_request:
paths:
- ".github/workflows/minio-interop.yml"
- "Cargo.lock"
- "Cargo.toml"
- "crates/ecstore/Cargo.toml"
- "crates/ecstore/src/io_support/rio.rs"
- "crates/ecstore/src/client/api_put_object_streaming.rs"
- "crates/ecstore/src/object_api/readers.rs"
- "crates/ecstore/src/set_disk/ops/object.rs"
- "crates/ecstore/src/sse/**"
- "crates/ecstore/src/store/object.rs"
- "crates/ecstore/tests/minio_generated_read_test.rs"
- "crates/kms/**"
- "crates/rio-v2/**"
- "rustfs/src/app/object_usecase.rs"
- "rustfs/Cargo.toml"
- "rustfs/src/storage/sse.rs"
schedule:
# Nightly at 03:17 UTC (offset from other nightly jobs).
- cron: "17 3 * * *"
@@ -41,13 +60,13 @@ permissions:
jobs:
minio-interop:
name: MinIO interop (EC + SSE read parity)
# Skip on forks: needs the repo's runners and is not a contributor gate.
# Skip on forks because this job depends on the repository's CI setup.
if: github.repository == 'rustfs/rustfs'
runs-on: ubuntu-latest
timeout-minutes: 40
env:
# Fixed 32-byte test KMS key baked into the fixture lab; not a secret.
RUSTFS_MINIO_STATIC_KMS_KEY_B64: IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g=
RUSTFS_MINIO_STATIC_KMS_KEY: minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g=
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -61,7 +80,10 @@ jobs:
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Generate real MinIO fixtures via Docker
run: bash crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh
run: bash crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh all
- name: Generate real RustFS beta.5 KMS fixture
run: uv run python crates/rio-v2/tests/minio_fixture_lab/capture_rustfs_beta5.py
- name: Run MinIO interop reader tests
run: |
+23
View File
@@ -132,6 +132,29 @@
"rust"
],
},
{
"name": "Debug executable target/debug/rustfs with sse-s3",
"type": "lldb",
"request": "launch",
"program": "${workspaceFolder}/target/debug/rustfs",
"args": [],
"cwd": "${workspaceFolder}",
"env": {
"RUSTFS_ACCESS_KEY": "rustfsadmin",
"RUSTFS_SECRET_KEY": "rustfsadmin",
"RUSTFS_VOLUMES": "./target/volumes/test{1...4}",
"RUSTFS_ADDRESS": ":9000",
"RUSTFS_CONSOLE_ENABLE": "true",
"RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
"RUSTFS_OBS_LOG_DIRECTORY": "./target/logs",
"RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true",
"RUSTFS_SSE_S3_MASTER_KEY": "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=",
"RUST_LOG": "rustfs=debug,ecstore=debug,s3s=debug,iam=debug"
},
"sourceLanguages": [
"rust"
]
},
{
"type": "lldb",
"request": "launch",
Generated
+50 -49
View File
@@ -3648,7 +3648,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "e2e_test"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -3685,7 +3685,6 @@ dependencies = [
"rustfs-protos",
"rustfs-rio",
"rustfs-signer",
"rustfs-utils",
"rustls",
"s3s",
"serde",
@@ -8825,7 +8824,7 @@ dependencies = [
[[package]]
name = "rustfs"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"aes-gcm",
"anyhow",
@@ -8953,13 +8952,14 @@ dependencies = [
"url",
"urlencoding",
"uuid",
"zeroize",
"zip",
"zstd",
]
[[package]]
name = "rustfs-audit"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"async-trait",
"chrono",
@@ -8981,7 +8981,7 @@ dependencies = [
[[package]]
name = "rustfs-checksums"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"base64-simd",
"bytes",
@@ -8996,7 +8996,7 @@ dependencies = [
[[package]]
name = "rustfs-common"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"chrono",
"metrics",
@@ -9011,7 +9011,7 @@ dependencies = [
[[package]]
name = "rustfs-concurrency"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"insta",
"rustfs-io-core",
@@ -9023,7 +9023,7 @@ dependencies = [
[[package]]
name = "rustfs-config"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"const-str",
"serde",
@@ -9032,7 +9032,7 @@ dependencies = [
[[package]]
name = "rustfs-credentials"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"base64-simd",
"hmac 0.13.0",
@@ -9045,7 +9045,7 @@ dependencies = [
[[package]]
name = "rustfs-crypto"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"aes-gcm",
"argon2",
@@ -9065,7 +9065,7 @@ dependencies = [
[[package]]
name = "rustfs-data-usage"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"async-trait",
"path-clean",
@@ -9076,7 +9076,7 @@ dependencies = [
[[package]]
name = "rustfs-ecstore"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"aes-gcm",
"arc-swap",
@@ -9094,6 +9094,7 @@ dependencies = [
"byteorder",
"bytes",
"bytesize",
"chacha20",
"chacha20poly1305",
"chrono",
"criterion",
@@ -9190,6 +9191,7 @@ dependencies = [
"urlencoding",
"uuid",
"xxhash-rust",
"zeroize",
]
[[package]]
@@ -9211,7 +9213,7 @@ dependencies = [
[[package]]
name = "rustfs-extension-schema"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"serde",
"serde_json",
@@ -9220,7 +9222,7 @@ dependencies = [
[[package]]
name = "rustfs-filemeta"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"arc-swap",
"byteorder",
@@ -9246,7 +9248,7 @@ dependencies = [
[[package]]
name = "rustfs-heal"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"async-trait",
"base64 0.22.1",
@@ -9276,7 +9278,7 @@ dependencies = [
[[package]]
name = "rustfs-iam"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"arc-swap",
"async-trait",
@@ -9313,7 +9315,7 @@ dependencies = [
[[package]]
name = "rustfs-io-core"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"bytes",
"memmap2",
@@ -9325,7 +9327,7 @@ dependencies = [
[[package]]
name = "rustfs-io-metrics"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"criterion",
"metrics",
@@ -9388,7 +9390,7 @@ dependencies = [
[[package]]
name = "rustfs-keystone"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"bytes",
"futures",
@@ -9414,7 +9416,7 @@ dependencies = [
[[package]]
name = "rustfs-kms"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"aes-gcm",
"arc-swap",
@@ -9445,7 +9447,7 @@ dependencies = [
[[package]]
name = "rustfs-lifecycle"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"async-trait",
"proptest",
@@ -9465,7 +9467,7 @@ dependencies = [
[[package]]
name = "rustfs-lock"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"async-trait",
"crossbeam-queue",
@@ -9486,7 +9488,7 @@ dependencies = [
[[package]]
name = "rustfs-log-analyzer"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"chrono",
"flate2",
@@ -9504,7 +9506,7 @@ dependencies = [
[[package]]
name = "rustfs-madmin"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"chrono",
"humantime",
@@ -9518,7 +9520,7 @@ dependencies = [
[[package]]
name = "rustfs-notify"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"arc-swap",
"async-trait",
@@ -9552,7 +9554,7 @@ dependencies = [
[[package]]
name = "rustfs-object-capacity"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"criterion",
"futures",
@@ -9570,7 +9572,7 @@ dependencies = [
[[package]]
name = "rustfs-object-data-cache"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"bytes",
"criterion",
@@ -9586,7 +9588,7 @@ dependencies = [
[[package]]
name = "rustfs-obs"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"chrono",
"crossbeam-channel",
@@ -9638,7 +9640,7 @@ dependencies = [
[[package]]
name = "rustfs-policy"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"async-trait",
"base64-simd",
@@ -9667,7 +9669,7 @@ dependencies = [
[[package]]
name = "rustfs-protocols"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"astral-tokio-tar",
"async-compression",
@@ -9726,7 +9728,7 @@ dependencies = [
[[package]]
name = "rustfs-protos"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"flatbuffers",
"prost 0.14.4",
@@ -9748,7 +9750,7 @@ dependencies = [
[[package]]
name = "rustfs-replication"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"byteorder",
"bytes",
@@ -9765,7 +9767,7 @@ dependencies = [
[[package]]
name = "rustfs-rio"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"aes-gcm",
"arc-swap",
@@ -9803,7 +9805,7 @@ dependencies = [
[[package]]
name = "rustfs-rio-v2"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"aes-gcm",
"bytes",
@@ -9825,14 +9827,14 @@ dependencies = [
[[package]]
name = "rustfs-s3-ops"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"rustfs-s3-types",
]
[[package]]
name = "rustfs-s3-types"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"serde",
"serde_json",
@@ -9840,7 +9842,7 @@ dependencies = [
[[package]]
name = "rustfs-s3select-api"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"async-trait",
"bytes",
@@ -9868,7 +9870,7 @@ dependencies = [
[[package]]
name = "rustfs-s3select-query"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"async-recursion",
"async-trait",
@@ -9884,7 +9886,7 @@ dependencies = [
[[package]]
name = "rustfs-scanner"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"async-trait",
"chrono",
@@ -9916,14 +9918,14 @@ dependencies = [
[[package]]
name = "rustfs-security-governance"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"thiserror 2.0.19",
]
[[package]]
name = "rustfs-signer"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"base64-simd",
"bytes",
@@ -9939,7 +9941,7 @@ dependencies = [
[[package]]
name = "rustfs-storage-api"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"async-trait",
"insta",
@@ -9953,7 +9955,7 @@ dependencies = [
[[package]]
name = "rustfs-targets"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"arc-swap",
"async-nats",
@@ -10006,7 +10008,7 @@ dependencies = [
[[package]]
name = "rustfs-test-utils"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"rustfs-data-usage",
"rustfs-ecstore",
@@ -10021,7 +10023,7 @@ dependencies = [
[[package]]
name = "rustfs-tls-runtime"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"arc-swap",
"metrics",
@@ -10041,7 +10043,7 @@ dependencies = [
[[package]]
name = "rustfs-trusted-proxies"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"async-trait",
"axum",
@@ -10077,7 +10079,7 @@ dependencies = [
[[package]]
name = "rustfs-utils"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"base64-simd",
"blake2",
@@ -10098,7 +10100,6 @@ dependencies = [
"netif",
"proptest",
"regex",
"reqwest",
"rustix",
"serde",
"sha1 0.11.0",
@@ -10117,7 +10118,7 @@ dependencies = [
[[package]]
name = "rustfs-zip"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
dependencies = [
"astral-tokio-tar",
"async-compression",
+48 -47
View File
@@ -69,7 +69,7 @@ edition = "2024"
license = "Apache-2.0"
repository = "https://github.com/rustfs/rustfs"
rust-version = "1.96.0"
version = "1.0.0-beta.11"
version = "1.0.0-beta.10"
homepage = "https://rustfs.com"
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -86,52 +86,52 @@ redundant_clone = "warn"
[workspace.dependencies]
# RustFS Internal Crates
rustfs = { path = "./rustfs", version = "1.0.0-beta.11" }
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.11" }
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.11" }
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.11" }
rustfs-common = { path = "crates/common", version = "1.0.0-beta.11" }
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.11" }
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.11" }
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.11" }
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.11" }
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.11" }
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.11" }
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.11" }
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.11" }
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.11" }
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.11" }
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.11" }
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.11" }
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.11" }
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.11" }
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.11" }
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.11" }
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.11" }
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.11" }
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.11" }
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.11" }
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.11" }
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.11" }
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.11" }
rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.11" }
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.11" }
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.11" }
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.11" }
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.11" }
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.11" }
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.11" }
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.11" }
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.11" }
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.11" }
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.11" }
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.11" }
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.11" }
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.11" }
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.11" }
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.11" }
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.11" }
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.11" }
rustfs = { path = "./rustfs", version = "1.0.0-beta.10" }
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.10" }
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.10" }
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.10" }
rustfs-common = { path = "crates/common", version = "1.0.0-beta.10" }
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.10" }
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.10" }
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.10" }
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.10" }
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.10" }
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.10" }
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.10" }
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.10" }
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.10" }
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-beta.10" }
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.10" }
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.10" }
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.10" }
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.10" }
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.10" }
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.10" }
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.10" }
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-beta.10" }
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-beta.10" }
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.10" }
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.10" }
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.10" }
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.10" }
rustfs-replication = { path = "crates/replication", version = "1.0.0-beta.10" }
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.10" }
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.10" }
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.10" }
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.10" }
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.10" }
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.10" }
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.10" }
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.10" }
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-beta.10" }
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.10" }
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.10" }
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.10" }
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.10" }
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-beta.10" }
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.10" }
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.10" }
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.10" }
# Async Runtime and Networking
async-channel = "2.5.0"
@@ -194,6 +194,7 @@ aes-gcm = { version = "=0.11.0" }
argon2 = { version = "=0.6.0-rc.8" }
blake2 = "=0.11.0-rc.6"
chacha20poly1305 = { version = "=0.11.0" }
chacha20 = { version = "=0.10.1", features = ["xchacha"] }
crc-fast = "1.10.0"
hmac = { version = "0.13.0" }
jsonwebtoken = { version = "10.4.0" }
+1 -1
View File
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
# Using specific version
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.11
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.10
```
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
+1 -1
View File
@@ -113,7 +113,7 @@ chown -R 10001:10001 data logs
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
# 使用指定版本运行
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.11
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.10
```
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
-30
View File
@@ -56,33 +56,3 @@ pub const DEFAULT_OBJECT_MMAP_READ_ENABLE: bool = true;
///
/// Prefer [`DEFAULT_OBJECT_MMAP_READ_ENABLE`].
pub const DEFAULT_OBJECT_ZERO_COPY_ENABLE: bool = DEFAULT_OBJECT_MMAP_READ_ENABLE;
/// Environment variable capping the byte length a single mmap-copy read may
/// materialize in memory.
///
/// The mmap-copy read path returns the whole requested range as one owned
/// allocation before the first byte is served. GET/heal shard reads request
/// the entire part span in one call, so for a large single-part object
/// (e.g. a multi-gigabyte non-multipart upload) an uncapped mmap-copy read
/// allocates the whole shard in memory — stalling first-byte latency past the
/// disk-read timeout and OOM-killing memory-limited deployments
/// (<https://github.com/rustfs/rustfs/issues/5123>). Reads longer than this
/// cap fall back to the bounded streaming reader instead.
///
/// - Purpose: Bound per-shard-read memory for mmap-based reads
/// - Acceptable values: byte count as an unsigned integer; `0` disables
/// mmap-copy for all non-empty reads (every read streams)
/// - Example: `export RUSTFS_OBJECT_MMAP_READ_MAX_LENGTH=8388608`
pub const ENV_OBJECT_MMAP_READ_MAX_LENGTH: &str = "RUSTFS_OBJECT_MMAP_READ_MAX_LENGTH";
/// Default mmap-copy read length cap: 32 MiB per shard read.
///
/// Large enough that typical multipart part shards (parts up to a few hundred
/// megabytes across the erasure set) keep the mmap fast path, small enough
/// that whole-part reads of huge single-part objects stream instead of
/// materializing gigabytes per shard.
///
/// The cap bounds memory per shard reader, so a single part read can still
/// materialize up to `data_shards x cap` bytes; raising the cap raises that
/// per-request bound proportionally.
pub const DEFAULT_OBJECT_MMAP_READ_MAX_LENGTH: usize = 32 * 1024 * 1024;
-1
View File
@@ -33,7 +33,6 @@ rustfs-config = { workspace = true, features = ["constants"] }
rustfs-ecstore.workspace = true
rustfs-data-usage.workspace = true
rustfs-rio.workspace = true
rustfs-utils = { workspace = true, features = ["egress"] }
flatbuffers.workspace = true
futures.workspace = true
rustfs-lock.workspace = true
+19 -9
View File
@@ -1068,7 +1068,8 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_s3() -> Result<(), B
#[tokio::test]
#[serial]
async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
async fn test_anonymous_post_object_bucket_default_sse_kms_requires_kms() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
{
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
@@ -1128,14 +1129,23 @@ async fn test_anonymous_post_object_uses_bucket_default_sse_kms() -> Result<(),
.send()
.await?;
assert_eq!(post_resp.status(), reqwest::StatusCode::NO_CONTENT);
let head = admin_client.head_object().bucket(bucket).key(object_key).send().await?;
assert_eq!(head.server_side_encryption().map(|value| value.as_str()), Some("aws:kms"));
let uploaded = admin_client.get_object().bucket(bucket).key(object_key).send().await?;
let uploaded = uploaded.body.collect().await?.into_bytes();
assert_eq!(uploaded.as_ref(), expected_body.as_slice());
let status = post_resp.status();
let response_body = post_resp.text().await?;
assert_eq!(status, reqwest::StatusCode::BAD_REQUEST);
assert!(
response_body.contains("configured KMS service"),
"unexpected response body: {response_body}"
);
assert!(
admin_client
.head_object()
.bucket(bucket)
.key(object_key)
.send()
.await
.is_err(),
"failed SSE-KMS POST must not create an object"
);
Ok(())
}
@@ -39,7 +39,6 @@ use local_ip_address::local_ip;
use reqwest::StatusCode;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
use s3s::Body;
use serde_json::Value;
use serial_test::serial;
@@ -71,11 +70,6 @@ fn target_arn(target_name: &str) -> String {
format!("arn:rustfs:sqs:{NOTIFY_REGION}:{target_name}:webhook")
}
fn endpoint_origin(endpoint: &str) -> Result<String, BoxError> {
let parsed = reqwest::Url::parse(endpoint)?;
Ok(parsed.origin().ascii_serialization())
}
// ---------------------------------------------------------------------------
// In-test HTTP event receiver
// ---------------------------------------------------------------------------
@@ -159,7 +153,7 @@ async fn spawn_event_collector() -> Result<(String, mpsc::UnboundedReceiver<Valu
let endpoint_ip = local_ip()?;
let (tx, rx) = mpsc::unbounded_channel();
let handle = serve_event_collector(listener, tx);
Ok((format!("http://{}/events", std::net::SocketAddr::new(endpoint_ip, port)), rx, handle))
Ok((format!("http://{endpoint_ip}.nip.io:{port}/events"), rx, handle))
}
struct HttpsEventCollector {
@@ -215,9 +209,9 @@ fn spawn_https_event_collector(ca_path: &Path) -> Result<HttpsEventCollector, Bo
listener.set_nonblocking(true)?;
let addr = listener.local_addr()?;
let endpoint_ip = local_ip()?;
let endpoint_host = endpoint_ip.to_string();
let endpoint_host = format!("{endpoint_ip}.nip.io");
let rcgen::CertifiedKey { cert, signing_key } = rcgen::generate_simple_self_signed(vec![endpoint_host])?;
let rcgen::CertifiedKey { cert, signing_key } = rcgen::generate_simple_self_signed(vec![endpoint_host.clone()])?;
std::fs::write(ca_path, cert.pem())?;
let cert_chain = vec![cert.der().clone()];
@@ -252,7 +246,7 @@ fn spawn_https_event_collector(ca_path: &Path) -> Result<HttpsEventCollector, Bo
});
Ok(HttpsEventCollector {
endpoint: format!("https://{}/events", std::net::SocketAddr::new(endpoint_ip, addr.port())),
endpoint: format!("https://{endpoint_host}:{}/events", addr.port()),
running,
handle: Some(handle),
events,
@@ -592,17 +586,11 @@ async fn test_https_webhook_target_delivers_event_with_notify_env_enabled() -> T
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_NOTIFY_ENABLE", "true")])
.await?;
let ca_path = Path::new(&env.temp_dir).join("https-webhook-ca.pem");
let mut collector = spawn_https_event_collector(&ca_path)?;
let allowed_origin = endpoint_origin(collector.endpoint())?;
env.start_rustfs_server_with_env(
vec![],
&[
("RUSTFS_NOTIFY_ENABLE", "true"),
(ENV_OUTBOUND_ALLOW_ORIGINS, allowed_origin.as_str()),
],
)
.await?;
let target = "peri1https";
let bucket = "peri1-https-events";
let key = "uploads/https.dat";
@@ -647,11 +635,9 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
init_logging();
let (endpoint, mut rx, handle) = spawn_event_collector().await?;
let allowed_origin = endpoint_origin(&endpoint)?;
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_OUTBOUND_ALLOW_ORIGINS, allowed_origin.as_str())])
.await?;
env.start_rustfs_server(vec![]).await?;
enable_notify_module(&env).await?;
let bucket = "peri1-events";
@@ -803,6 +789,15 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
enable_notify_module(&env).await?;
let bucket = "peri1-redeliver";
let target = "peri1redeliver";
let client = env.create_s3_client();
client.create_bucket().bucket(bucket).send().await?;
// Configure the target while its endpoint is reachable so activation and
// ARN registration complete deterministically. Registering against a dead
// endpoint stalls behind the reachability probe's timeout and flakes the
@@ -811,21 +806,10 @@ async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult {
let listener = TcpListener::bind("0.0.0.0:0").await?;
let port = listener.local_addr()?.port();
let endpoint_ip = local_ip()?;
let endpoint = format!("http://{}/events", std::net::SocketAddr::new(endpoint_ip, port));
let allowed_origin = endpoint_origin(&endpoint)?;
let endpoint = format!("http://{endpoint_ip}.nip.io:{port}/events");
let (setup_tx, _setup_rx) = mpsc::unbounded_channel();
let setup_handle = serve_event_collector(listener, setup_tx);
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_with_env(vec![], &[(ENV_OUTBOUND_ALLOW_ORIGINS, allowed_origin.as_str())])
.await?;
enable_notify_module(&env).await?;
let bucket = "peri1-redeliver";
let target = "peri1redeliver";
let client = env.create_s3_client();
client.create_bucket().bucket(bucket).send().await?;
configure_webhook_target(&env, target, &endpoint).await?;
wait_for_target_registered(&env, target).await?;
put_notification_config(&client, bucket, target, "uploads/", ".dat").await?;
+19 -52
View File
@@ -18,7 +18,6 @@ use http::header::{CONTENT_TYPE, HOST};
use reqwest::StatusCode;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::{pre_sign_v4, sign_v4};
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
use s3s::Body;
use serial_test::serial;
use std::collections::HashMap;
@@ -50,7 +49,7 @@ fn find_header_terminator(buf: &[u8]) -> Option<usize> {
async fn read_http_request(
stream: &mut tokio::net::TcpStream,
) -> Result<(String, HashMap<String, String>, Vec<u8>), Box<dyn Error + Send + Sync>> {
) -> Result<(HashMap<String, String>, Vec<u8>), Box<dyn Error + Send + Sync>> {
let mut buffer = Vec::new();
let mut chunk = [0_u8; 4096];
@@ -68,7 +67,7 @@ async fn read_http_request(
let header_bytes = &buffer[..header_end];
let header_text = std::str::from_utf8(header_bytes)?;
let mut lines = header_text.split("\r\n");
let request_line = lines.next().ok_or("missing request line")?.to_string();
let _request_line = lines.next().ok_or("missing request line")?;
let mut headers = HashMap::new();
for line in lines {
if line.is_empty() {
@@ -80,9 +79,8 @@ async fn read_http_request(
let content_length = headers
.get("content-length")
.map(|value| value.parse::<usize>())
.transpose()?
.unwrap_or_default();
.ok_or("missing content-length header")?
.parse::<usize>()?;
let body_offset = header_end + 4;
while buffer.len().saturating_sub(body_offset) < content_length {
let read = stream.read(&mut chunk).await?;
@@ -92,7 +90,7 @@ async fn read_http_request(
buffer.extend_from_slice(&chunk[..read]);
}
Ok((request_line, headers, buffer[body_offset..body_offset + content_length].to_vec()))
Ok((headers, buffer[body_offset..body_offset + content_length].to_vec()))
}
async fn spawn_object_lambda_webhook_server() -> Result<
@@ -132,17 +130,9 @@ async fn spawn_object_lambda_webhook_server_with_response(
let handle = tokio::spawn(async move {
loop {
let (mut stream, _) = listener.accept().await?;
let Ok(Ok((request_line, headers, body))) = timeout(Duration::from_secs(2), read_http_request(&mut stream)).await
else {
let Ok(Ok((headers, body))) = timeout(Duration::from_secs(2), read_http_request(&mut stream)).await else {
continue;
};
if request_line == "HEAD / HTTP/1.1" {
stream
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
.await?;
stream.shutdown().await?;
continue;
}
let payload: serde_json::Value = serde_json::from_slice(&body)?;
let output_route = payload["getObjectContext"]["outputRoute"]
@@ -422,33 +412,9 @@ async fn wait_for_target_absence(
Err(format!("target {target_name} remained visible in admin APIs; targets={last_targets}, arns={last_arns:?}").into())
}
fn endpoint_origin(endpoint: &str) -> Result<String, Box<dyn Error + Send + Sync>> {
Ok(reqwest::Url::parse(endpoint)?.origin().ascii_serialization())
}
async fn start_rustfs_server_for_endpoint(
env: &mut RustFSTestEnvironment,
endpoint: &str,
) -> Result<(), Box<dyn Error + Send + Sync>> {
let origin = endpoint_origin(endpoint)?;
env.start_rustfs_server_with_env(
vec![],
&[
(ENV_OUTBOUND_ALLOW_ORIGINS, origin.as_str()),
("RUSTFS_NOTIFY_ENABLE", "true"),
],
)
.await
}
async fn restart_rustfs_server(env: &mut RustFSTestEnvironment, endpoint: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
let origin = endpoint_origin(endpoint)?;
async fn restart_rustfs_server(env: &mut RustFSTestEnvironment) -> Result<(), Box<dyn Error + Send + Sync>> {
env.stop_server();
env.start_rustfs_server_without_cleanup_with_env(&[
(ENV_OUTBOUND_ALLOW_ORIGINS, origin.as_str()),
("RUSTFS_NOTIFY_ENABLE", "true"),
])
.await
env.start_rustfs_server_without_cleanup(vec![]).await
}
async fn spawn_http_origin_probe_server() -> Result<
@@ -555,7 +521,7 @@ async fn test_notification_target_persists_across_restart_and_delete() -> Result
let (webhook_url, _request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?;
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
env.start_rustfs_server(vec![]).await?;
let target_name = "restart-target";
configure_webhook_target(&env, target_name, &webhook_url, "secret-token").await?;
@@ -569,7 +535,7 @@ async fn test_notification_target_persists_across_restart_and_delete() -> Result
"target ARN missing after initial configure: {visible_arns:?}"
);
restart_rustfs_server(&mut env, &webhook_url).await?;
restart_rustfs_server(&mut env).await?;
let (targets_after_restart, arns_after_restart) = wait_for_target_visibility(&env, target_name).await?;
assert!(notification_target_is_listed(&targets_after_restart, target_name));
@@ -590,7 +556,7 @@ async fn test_notification_target_persists_across_restart_and_delete() -> Result
"target ARN still visible after delete: {arns_after_delete:?}"
);
restart_rustfs_server(&mut env, &webhook_url).await?;
restart_rustfs_server(&mut env).await?;
let (targets_after_delete_restart, arns_after_delete_restart) = wait_for_target_absence(&env, target_name).await?;
assert!(!notification_target_is_listed(&targets_after_delete_restart, target_name));
@@ -615,7 +581,8 @@ async fn test_notification_target_with_path_is_online_via_transport_probe() -> R
let (webhook_url, mut probe_rx, probe_handle) = spawn_http_origin_probe_server().await?;
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_NOTIFY_ENABLE", "true")])
.await?;
let target_name = "path-probe";
configure_webhook_target(&env, target_name, &webhook_url, "secret-token").await?;
@@ -648,7 +615,7 @@ async fn test_get_object_lambda_accepts_presigned_requests() -> Result<(), Box<d
let (webhook_url, request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?;
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
env.start_rustfs_server(vec![]).await?;
let bucket = "object-lambda-e2e-presigned";
let key = "input.txt";
@@ -689,7 +656,7 @@ async fn test_get_object_lambda_accepts_named_webhook_target_arn() -> Result<(),
let (webhook_url, request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?;
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
env.start_rustfs_server(vec![]).await?;
let bucket = "object-lambda-e2e-named-target";
let key = "input.txt";
@@ -729,7 +696,7 @@ async fn test_get_object_lambda_invokes_runtime_webhook_target() -> Result<(), B
let (webhook_url, request_rx, webhook_handle) = spawn_object_lambda_webhook_server().await?;
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
env.start_rustfs_server(vec![]).await?;
let bucket = "object-lambda-e2e";
let key = "input.txt";
@@ -810,7 +777,7 @@ async fn test_get_object_lambda_passthroughs_non_success_webhook_response() -> R
.await?;
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
env.start_rustfs_server(vec![]).await?;
let bucket = "object-lambda-e2e-failure";
let key = "input.txt";
@@ -865,7 +832,7 @@ async fn test_get_object_lambda_rejects_success_response_without_auth_headers()
.await?;
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
env.start_rustfs_server(vec![]).await?;
let bucket = "object-lambda-e2e-missing-auth";
let key = "input.txt";
@@ -912,7 +879,7 @@ async fn test_get_object_lambda_rejects_success_response_with_mismatched_auth_he
.await?;
let mut env = RustFSTestEnvironment::new().await?;
start_rustfs_server_for_endpoint(&mut env, &webhook_url).await?;
env.start_rustfs_server(vec![]).await?;
let bucket = "object-lambda-e2e-mismatched-auth";
let key = "input.txt";
@@ -4048,23 +4048,17 @@ async fn test_site_replication_allows_private_ca_https_with_ca_cert_pem_real_dua
#[tokio::test]
#[serial]
async fn test_site_replication_resync_lifecycle_survives_real_server_restart() -> Result<(), Box<dyn Error + Send + Sync>> {
async fn test_site_replication_resync_start_cancel_restart_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let resync_process_env = [
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", "true"),
// Verbose server logging can block startup when this focused test is run
// through a captured test process rather than nextest.
("RUST_LOG", "error"),
];
let mut source_env = RustFSTestEnvironment::new().await?;
source_env.capture_log_path = Some(format!("{}/server.log", source_env.temp_dir));
source_env.start_rustfs_server_with_env(vec![], &resync_process_env).await?;
source_env
.start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)
.await?;
let mut target_env = RustFSTestEnvironment::new().await?;
target_env.capture_log_path = Some(format!("{}/server.log", target_env.temp_dir));
target_env
.start_rustfs_server_without_cleanup_with_env(&resync_process_env)
.start_rustfs_server_without_cleanup_with_env(LOOPBACK_REPLICATION_TARGET_ENV)
.await?;
let source_bucket = "site-repl-resync-src";
@@ -4112,12 +4106,12 @@ async fn test_site_replication_resync_lifecycle_survives_real_server_restart() -
wait_for_bucket_on_target(&source_client, source_bucket).await?;
let target_arn = wait_for_remote_target_arn(&source_env, source_bucket).await?;
for idx in 0..96 {
for idx in 0..32 {
source_client
.put_object()
.bucket(source_bucket)
.key(format!("resync-object-{idx:02}"))
.body(ByteStream::from(vec![b'x'; 512 * 1024]))
.body(ByteStream::from(vec![b'x'; 256 * 1024]))
.send()
.await?;
}
@@ -4125,31 +4119,18 @@ async fn test_site_replication_resync_lifecycle_survives_real_server_restart() -
let started = site_replication_resync_op(&source_env, "start", &remote_peer).await?;
assert_eq!(started.status, "success", "unexpected start result: {:?}", started);
assert!(
started.buckets.iter().any(|bucket| {
bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "running" | "completed" | "success")
}),
started
.buckets
.iter()
.any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "success")),
"source bucket start status missing: {:?}",
started
);
assert!(!started.resync_id.is_empty(), "start response omitted the resync id: {:?}", started);
let started_reset_id = started.resync_id.clone();
assert!(
matches!(started.state.as_str(), "pending" | "running"),
"the fixture must keep the first generation active long enough to test duplicate start: {:?}",
started
);
let duplicate_err = site_replication_resync_op(&source_env, "start", &remote_peer)
.await
.expect_err("duplicate start must be rejected while a generation is active");
assert!(
duplicate_err.to_string().contains("already active"),
"unexpected duplicate start error: {duplicate_err}"
);
let canceled = site_replication_resync_op(&source_env, "cancel", &remote_peer).await?;
assert_eq!(canceled.status, "success", "unexpected cancel result: {:?}", canceled);
assert_eq!(canceled.state, "canceled");
assert!(
canceled
.buckets
@@ -4158,56 +4139,34 @@ async fn test_site_replication_resync_lifecycle_survives_real_server_restart() -
"source bucket cancel status missing: {:?}",
canceled
);
let canceled_again = site_replication_resync_op(&source_env, "cancel", &remote_peer).await?;
assert_eq!(canceled_again.resync_id, canceled.resync_id, "repeated cancel must be idempotent");
assert_eq!(canceled_again.state, "canceled");
let canceled_target =
wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| target.status == "Canceled").await?;
assert_eq!(canceled_target.status, "Canceled");
assert_eq!(canceled_target.reset_id, started_reset_id);
let restarted = site_replication_resync_op(&source_env, "start", &remote_peer).await?;
assert_eq!(restarted.status, "success", "unexpected restart result: {:?}", restarted);
assert_ne!(restarted.resync_id, started_reset_id);
assert!(
matches!(restarted.state.as_str(), "pending" | "running"),
"the second generation must be active before the process restart: {:?}",
restarted
);
let restarted_reset_id = restarted.resync_id.clone();
source_env.restart_server_preserving_data(vec![], &resync_process_env).await?;
wait_for_site_replication_enabled(&source_env, 2).await?;
let after_restart = site_replication_resync_op(&source_env, "status", &remote_peer).await?;
assert_eq!(
after_restart.resync_id, restarted_reset_id,
"server restart changed the durable resync id"
);
assert_eq!(after_restart.generation, restarted.generation);
assert_eq!(after_restart.created_at, restarted.created_at);
assert!(
matches!(after_restart.state.as_str(), "pending" | "running" | "completed" | "failed"),
"unexpected recovered lifecycle state: {:?}",
after_restart
);
assert!(
after_restart.buckets.iter().any(|bucket| bucket.bucket == source_bucket),
"durable status lost the source bucket after restart: {:?}",
after_restart
.buckets
.iter()
.any(|bucket| bucket.bucket == source_bucket && matches!(bucket.status.as_str(), "started" | "success")),
"source bucket restart status missing: {:?}",
restarted
);
let restart_snapshot = get_replication_reset_status(&source_env, source_bucket, &target_arn).await?;
let restarted_target = wait_for_replication_reset_target(&source_env, source_bucket, &target_arn, |target| {
target.reset_id == restarted_reset_id
!target.reset_id.is_empty() && target.reset_id != started_reset_id
})
.await
.map_err(|err| {
format!(
"restart ids: start={} restart={} snapshot={:?}; {err}",
started_reset_id, restarted_reset_id, restart_snapshot.targets
started_reset_id, restarted.resync_id, restart_snapshot.targets
)
})?;
assert_eq!(restarted_target.reset_id, restarted_reset_id);
assert_ne!(restarted_target.reset_id, started_reset_id);
Ok(())
}
+2
View File
@@ -126,6 +126,8 @@ rustfs-madmin.workspace = true
reqwest = { workspace = true }
aes-gcm = { workspace = true, features = ["rand_core"] }
chacha20poly1305.workspace = true
chacha20.workspace = true
zeroize.workspace = true
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
urlencoding = { workspace = true }
smallvec = { workspace = true, features = ["serde"] }
+11 -3
View File
@@ -361,9 +361,10 @@ pub mod notification {
pub mod object {
pub use crate::object_api::{
BLOCK_SIZE_V2, ERASURE_ALGORITHM, GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource,
GetObjectReader, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader, StreamConsumer,
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
GetObjectReader, GetObjectSse, ObjectInfo, ObjectMutationHook, ObjectOptions, PutObjReader, RangedDecompressReader,
StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
unregister_object_mutation_hook,
};
pub use crate::store::PreparedGetObjectReader;
}
@@ -398,6 +399,13 @@ pub mod set_disk {
pub use crate::set_disk::{DEFAULT_READ_BUFFER_SIZE, SetDisks, get_lock_acquire_timeout, is_valid_storage_class};
}
pub mod sse {
pub use crate::sse::{
ManagedDekProvider, ManagedSseScheme, PersistedEncryptionError, PersistedManagedEncryption,
classify_persisted_managed_encryption, decrypt_minio_static_kms_dek,
};
}
pub mod store_list {
pub use crate::store::list_objects::{ListPathOptions, max_keys_plus_one};
}
@@ -1625,6 +1625,7 @@ mod tests {
..Default::default()
},
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
})
}
@@ -657,6 +657,7 @@ mod tests {
stream: Box::new(r),
object_info: Default::default(),
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
});
@@ -687,6 +688,7 @@ mod tests {
stream: Box::new(r),
object_info: Default::default(),
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
});
let buf = read_multipart_part(&mut reader, 100).await.unwrap();
@@ -153,6 +153,7 @@ pub fn new_getobjectreader<'a>(
object_info: oi.clone(),
stream: Box::new(input_reader),
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
};
r
@@ -166,6 +167,7 @@ pub fn new_getobjectreader<'a>(
object_info: oi.clone(),
stream: Box::new(input_reader),
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
});
@@ -219,62 +219,31 @@ impl PeerRestClient {
recovery_running: Arc::new(AtomicBool::new(false)),
}
}
fn build_clients_from_slots(
slots: Vec<(String, Option<String>, bool)>,
) -> (Vec<Option<Self>>, Vec<Option<Self>>, Vec<String>) {
let mut remote = Vec::with_capacity(slots.len().saturating_sub(1));
let mut all = vec![None; slots.len()];
let mut remote_topology_hosts = Vec::with_capacity(slots.len().saturating_sub(1));
for (idx, (peer_host_port, grid_host, is_local)) in slots.into_iter().enumerate() {
if is_local {
continue;
}
let client = match grid_host {
Some(grid_host) => match XHost::try_from(peer_host_port.clone()) {
Ok(host) => Some(PeerRestClient::new(host, grid_host)),
Err(err) => {
warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}");
None
}
},
None => {
warn!(peer = %peer_host_port, "grid host is missing while constructing peer client");
None
}
};
all[idx] = client.clone();
remote.push(client);
remote_topology_hosts.push(peer_host_port);
}
(remote, all, remote_topology_hosts)
}
pub async fn new_clients(eps: EndpointServerPools) -> (Vec<Option<Self>>, Vec<Option<Self>>) {
let (remote, all, _) = Self::new_clients_with_topology(eps).await;
(remote, all)
}
pub async fn new_clients_with_topology(eps: EndpointServerPools) -> (Vec<Option<Self>>, Vec<Option<Self>>, Vec<String>) {
if !runtime_sources::setup_is_dist_erasure().await {
return (Vec::new(), Vec::new(), Vec::new());
return (Vec::new(), Vec::new());
}
let (remote, all, remote_topology_hosts) = Self::build_clients_from_slots(eps.peer_grid_host_slots_sorted());
let eps = eps.clone();
let hosts = eps.hosts_sorted();
let mut remote = Vec::with_capacity(hosts.len());
let mut all = vec![None; hosts.len()];
for (i, hs_host) in hosts.iter().enumerate() {
if let Some(host) = hs_host
&& let Some(grid_host) = eps.find_grid_hosts_from_peer(host)
{
let client = PeerRestClient::new(host.clone(), grid_host);
all[i] = Some(client.clone());
remote.push(Some(client));
}
}
if all.len() != remote.len() + 1 {
warn!(
all_hosts = all.len(),
remote_slots = remote.len(),
"Expected number of all hosts to be remote slots + local node"
);
warn!("Expected number of all hosts ({}) to be remote +1 ({})", all.len(), remote.len());
}
(remote, all, remote_topology_hosts)
(remote, all)
}
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
@@ -1567,36 +1536,6 @@ mod tests {
)
}
#[test]
fn build_clients_from_slots_preserves_missing_remote_topology_slots() {
let slots = vec![
("127.0.0.1:9000".to_string(), None, true),
("127.0.0.1:9001".to_string(), Some("http://127.0.0.1:9001".to_string()), false),
("127.0.0.1:notaport".to_string(), Some("http://127.0.0.1:notaport".to_string()), false),
("127.0.0.1:9003".to_string(), None, false),
];
let (remote, all, remote_topology_hosts) = PeerRestClient::build_clients_from_slots(slots);
assert_eq!(remote.len(), 3, "local node is excluded but remote slots are not compacted away");
assert_eq!(all.len(), 4, "all slots preserve the sorted cluster topology shape");
assert_eq!(
remote_topology_hosts,
vec![
"127.0.0.1:9001".to_string(),
"127.0.0.1:notaport".to_string(),
"127.0.0.1:9003".to_string()
]
);
assert!(remote[0].is_some(), "valid remote peer should get a client");
assert!(remote[1].is_none(), "unparseable remote peer should remain observable as a missing slot");
assert!(remote[2].is_none(), "missing grid host should remain observable as a missing slot");
assert!(all[0].is_none(), "local node is represented by the local server_info row");
assert!(all[1].is_some());
assert!(all[2].is_none());
assert!(all[3].is_none());
}
#[test]
fn scanner_activity_requires_restart_safe_peer_identity() {
let missing_instance = ScannerActivityResponse {
+2
View File
@@ -2089,6 +2089,7 @@ mod tests {
}),
object_info: self.object_info(bucket, object),
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
})
}
@@ -3315,6 +3316,7 @@ mod tests {
stream: Box::new(Cursor::new(data)),
object_info,
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
})
}
+1
View File
@@ -1240,6 +1240,7 @@ mod tests {
stream: Box::new(Cursor::new(raw_payload.clone())),
object_info: object_info.clone(),
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
};
@@ -15,11 +15,7 @@
use crate::cluster::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend_cached};
use crate::error::{Error, Result};
use crate::{
disk::endpoint::{Endpoint, EndpointType},
layout::endpoints::EndpointServerPools,
runtime::sources as runtime_sources,
};
use crate::{disk::endpoint::Endpoint, runtime::sources as runtime_sources};
use crate::data_usage::load_data_usage_cache;
use crate::storage_api_contracts::admin::StorageAdminApi;
@@ -295,20 +291,6 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
let after4 = OffsetDateTime::now_utc();
warn!("backend_info end {:?}", after4 - after3);
if let Some(endpoints) = runtime_sources::endpoint_pools() {
let (added, report) = reconcile_servers_with_endpoint_topology(&mut servers, &endpoints);
if added > 0 || !report.is_complete() {
warn!(
event = "admin_v3_info_topology_incomplete",
synthesized_servers = added,
expected_drives = report.expected_drives,
observed_drives = report.observed_drives,
missing_drives = report.missing_drive_ids.len(),
duplicate_drives = report.duplicate_drive_ids.len(),
"admin v3 server_info reconciled endpoint topology before computing backend counters"
);
}
}
let mut all_disks: Vec<Disk> = Vec::new();
for server in servers.iter() {
@@ -415,228 +397,6 @@ fn get_online_offline_disks_stats(disks_info: &[Disk]) -> (BackendDisks, Backend
(BackendDisks(online_disks), BackendDisks(offline_disks), BackendDisks(unknown_disks))
}
#[derive(Debug, Default, PartialEq, Eq)]
struct TopologyCompletenessReport {
expected_drives: usize,
observed_drives: usize,
missing_drive_ids: Vec<TopologyDriveKey>,
duplicate_drive_ids: Vec<TopologyDriveKey>,
}
impl TopologyCompletenessReport {
fn is_complete(&self) -> bool {
self.expected_drives == self.observed_drives && self.missing_drive_ids.is_empty() && self.duplicate_drive_ids.is_empty()
}
}
#[derive(Debug)]
struct TopologyMember {
display_endpoint: String,
disks: Vec<Disk>,
drive_keys: Vec<TopologyDriveKey>,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
struct TopologyDriveKey {
pool_index: i32,
set_index: i32,
disk_index: i32,
member_index: usize,
}
#[derive(Debug, Default)]
struct EndpointTopology {
members: Vec<TopologyMember>,
aliases: HashMap<String, usize>,
expected_drive_ids: HashSet<TopologyDriveKey>,
}
impl EndpointTopology {
fn from_endpoint_pools(endpoints: &EndpointServerPools) -> Self {
let mut topology = Self::default();
let mut by_host_port = HashMap::new();
let mut display_aliases: HashMap<String, Option<usize>> = HashMap::new();
for pool in endpoints.as_ref() {
for ep in pool.endpoints.as_ref() {
if ep.get_type() != EndpointType::Url {
continue;
}
let host_port = ep.host_port();
if host_port.is_empty() {
continue;
}
let member_index = match by_host_port.get(&host_port).copied() {
Some(index) => index,
None => {
let display_endpoint = ep.url.host_str().map(str::to_owned).unwrap_or_else(|| host_port.clone());
let index = topology.members.len();
topology.members.push(TopologyMember {
display_endpoint,
disks: Vec::new(),
drive_keys: Vec::new(),
});
by_host_port.insert(host_port.clone(), index);
index
}
};
topology.aliases.entry(host_port.clone()).or_insert(member_index);
topology.aliases.entry(ep.to_string()).or_insert(member_index);
if let Some(display_endpoint) = ep.url.host_str() {
match display_aliases.entry(display_endpoint.to_owned()) {
std::collections::hash_map::Entry::Vacant(entry) => {
entry.insert(Some(member_index));
}
std::collections::hash_map::Entry::Occupied(mut entry) => {
if entry.get().is_some_and(|index| index != member_index) {
entry.insert(None);
}
}
}
}
let drive_key = TopologyDriveKey {
pool_index: ep.pool_idx,
set_index: ep.set_idx,
disk_index: ep.disk_idx,
member_index,
};
topology.expected_drive_ids.insert(drive_key.clone());
topology.members[member_index].drive_keys.push(drive_key);
topology.members[member_index].disks.push(Disk {
endpoint: ep.to_string(),
state: ITEM_UNKNOWN.to_string(),
pool_index: ep.pool_idx,
set_index: ep.set_idx,
disk_index: ep.disk_idx,
..Default::default()
});
}
}
for (display_endpoint, member_index) in display_aliases {
if let Some(member_index) = member_index {
topology.aliases.entry(display_endpoint).or_insert(member_index);
}
}
topology
}
fn is_empty(&self) -> bool {
self.members.is_empty()
}
fn observe_servers(&self, servers: &[ServerProperties]) -> (Vec<bool>, HashMap<TopologyDriveKey, usize>) {
let mut observed_members = vec![false; self.members.len()];
let mut observed_drive_counts = HashMap::with_capacity(self.expected_drive_ids.len());
for server in servers {
if let Some(member_index) = self.member_index_from_endpoint(&server.endpoint) {
observed_members[member_index] = true;
}
for disk in &server.disks {
if let Some(member_index) = self.member_index_from_endpoint(&disk.endpoint) {
observed_members[member_index] = true;
let drive_key = TopologyDriveKey {
pool_index: disk.pool_index,
set_index: disk.set_index,
disk_index: disk.disk_index,
member_index,
};
if self.expected_drive_ids.contains(&drive_key) {
*observed_drive_counts.entry(drive_key).or_insert(0) += 1;
}
}
}
}
(observed_members, observed_drive_counts)
}
fn member_index_from_endpoint(&self, endpoint: &str) -> Option<usize> {
if let Some(index) = self.aliases.get(endpoint) {
return Some(*index);
}
Endpoint::try_from(endpoint)
.ok()
.and_then(|ep| self.aliases.get(&ep.host_port()).copied())
}
fn report_from_counts(&self, observed_drive_counts: HashMap<TopologyDriveKey, usize>) -> TopologyCompletenessReport {
let mut missing_drive_ids = Vec::new();
let mut duplicate_drive_ids = Vec::new();
for id in &self.expected_drive_ids {
match observed_drive_counts.get(id).copied().unwrap_or(0) {
0 => missing_drive_ids.push(id.clone()),
1 => {}
_ => duplicate_drive_ids.push(id.clone()),
}
}
missing_drive_ids.sort();
duplicate_drive_ids.sort();
TopologyCompletenessReport {
expected_drives: self.expected_drive_ids.len(),
observed_drives: observed_drive_counts.values().sum(),
missing_drive_ids,
duplicate_drive_ids,
}
}
}
fn reconcile_servers_with_endpoint_topology(
servers: &mut Vec<ServerProperties>,
endpoints: &EndpointServerPools,
) -> (usize, TopologyCompletenessReport) {
let topology = EndpointTopology::from_endpoint_pools(endpoints);
if topology.is_empty() {
return (0, TopologyCompletenessReport::default());
}
let (observed_members, mut observed_drive_counts) = topology.observe_servers(servers);
let mut missing: Vec<_> = topology
.members
.iter()
.enumerate()
.filter(|(index, _)| !observed_members[*index])
.map(|(_, member)| {
for drive_key in &member.drive_keys {
*observed_drive_counts.entry(drive_key.clone()).or_insert(0) += 1;
}
ServerProperties {
endpoint: member.display_endpoint.clone(),
state: ITEM_UNKNOWN.to_string(),
disks: member.disks.clone(),
..Default::default()
}
})
.collect();
missing.sort_by(|a, b| a.endpoint.cmp(&b.endpoint));
let added = missing.len();
servers.extend(missing);
let report = topology.report_from_counts(observed_drive_counts);
(added, report)
}
fn server_topology_completeness_report(
servers: &[ServerProperties],
endpoints: &EndpointServerPools,
) -> TopologyCompletenessReport {
let topology = EndpointTopology::from_endpoint_pools(endpoints);
if topology.is_empty() {
return TopologyCompletenessReport::default();
}
let (_, observed_drive_counts) = topology.observe_servers(servers);
topology.report_from_counts(observed_drive_counts)
}
async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32, ErasureSetInfo>>> {
let Some(store) = runtime_sources::object_store_handle() else {
return Err(Error::other("ServerNotInitialized"));
@@ -689,16 +449,12 @@ pub fn get_commit_id() -> String {
mod tests {
use serial_test::serial;
use crate::layout::{
endpoint::Endpoint,
endpoints::{EndpointServerPools, Endpoints, PoolEndpoints},
};
use crate::runtime::sources as runtime_sources;
use rustfs_madmin::{Disk, ITEM_OFFLINE, ITEM_ONLINE, ITEM_UNKNOWN, ServerProperties};
use rustfs_madmin::{Disk, ITEM_OFFLINE, ITEM_UNKNOWN};
use super::{
DATA_USAGE_UNAVAILABLE_ERROR, apply_data_usage_result, get_local_server_property, get_online_offline_disks_stats,
get_server_info, reconcile_servers_with_endpoint_topology, server_topology_completeness_report,
get_server_info,
};
fn disk_with_state(endpoint: &str, state: &str) -> Disk {
@@ -709,50 +465,6 @@ mod tests {
}
}
fn topology_endpoint(host: &str, pool_index: usize, set_index: usize, disk_index: usize) -> Endpoint {
topology_endpoint_url(format!("http://{host}:9000/data{disk_index}").as_str(), pool_index, set_index, disk_index)
}
fn topology_endpoint_url(url: &str, pool_index: usize, set_index: usize, disk_index: usize) -> Endpoint {
let mut endpoint = Endpoint::try_from(url).expect("URL endpoint should parse");
endpoint.set_pool_index(pool_index);
endpoint.set_set_index(set_index);
endpoint.set_disk_index(disk_index);
endpoint
}
fn topology_with_hosts(hosts: &[&str]) -> EndpointServerPools {
let endpoints: Vec<Endpoint> = hosts
.iter()
.enumerate()
.map(|(disk_index, host)| topology_endpoint(host, 0, 0, disk_index))
.collect();
EndpointServerPools::from(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: hosts.len(),
endpoints: Endpoints::from(endpoints),
cmd_line: String::new(),
platform: String::new(),
}])
}
fn server_with_disk(host: &str, disk_index: i32, state: &str) -> ServerProperties {
ServerProperties {
endpoint: host.to_string(),
state: ITEM_ONLINE.to_string(),
disks: vec![Disk {
endpoint: format!("http://{host}:9000/data{disk_index}"),
state: state.to_string(),
pool_index: 0,
set_index: 0,
disk_index,
..Default::default()
}],
..Default::default()
}
}
#[test]
fn disk_stats_split_unknown_into_its_own_bucket() {
// A member whose properties RPC could not be answered contributes
@@ -781,93 +493,6 @@ mod tests {
);
}
#[test]
fn topology_reconcile_synthesizes_missing_member_as_unknown() {
let endpoints = topology_with_hosts(&["rustfs-1", "rustfs-2", "rustfs-3", "rustfs-4"]);
let mut servers = vec![
server_with_disk("rustfs-1", 0, "ok"),
server_with_disk("rustfs-2", 1, "ok"),
server_with_disk("rustfs-3", 2, "ok"),
];
let (added, report) = reconcile_servers_with_endpoint_topology(&mut servers, &endpoints);
let (_, _, unknown) =
get_online_offline_disks_stats(&servers.iter().flat_map(|server| server.disks.clone()).collect::<Vec<_>>());
assert_eq!(added, 1, "the missing fourth topology member must be synthesized");
assert!(report.is_complete(), "synthesized drives should complete the topology report");
assert_eq!(servers.len(), 4, "v3 server list must preserve topology membership length");
let synthesized = servers
.iter()
.find(|server| server.endpoint == "rustfs-4")
.expect("missing member should be present");
assert_eq!(synthesized.state, ITEM_UNKNOWN);
assert_eq!(synthesized.disks.len(), 1);
assert_eq!(synthesized.disks[0].endpoint, "http://rustfs-4:9000/data3");
assert_eq!(synthesized.disks[0].disk_index, 3);
assert_eq!(unknown.sum(), 1, "the synthesized drive must land in unknownDisks");
}
#[test]
fn topology_reconcile_does_not_duplicate_existing_synthesized_rows() {
let endpoints = topology_with_hosts(&["rustfs-1", "rustfs-2"]);
let mut servers = vec![
server_with_disk("rustfs-1", 0, "ok"),
server_with_disk("rustfs-2", 1, ITEM_UNKNOWN),
];
servers[1].state = ITEM_UNKNOWN.to_string();
let (added, report) = reconcile_servers_with_endpoint_topology(&mut servers, &endpoints);
assert_eq!(
added, 0,
"an existing unknown/degraded/offline row with topology drives already represents the member"
);
assert!(report.is_complete(), "existing synthesized rows should already complete the topology");
assert_eq!(servers.len(), 2);
}
#[test]
fn topology_reconcile_does_not_over_match_ambiguous_host_only_aliases() {
let endpoints = EndpointServerPools::from(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 2,
endpoints: Endpoints::from(vec![
topology_endpoint_url("http://rustfs-1:9000/data0", 0, 0, 0),
topology_endpoint_url("http://rustfs-1:9001/data1", 0, 0, 1),
]),
cmd_line: String::new(),
platform: String::new(),
}]);
let mut servers = vec![server_with_disk("rustfs-1", 0, "ok")];
let (added, report) = reconcile_servers_with_endpoint_topology(&mut servers, &endpoints);
assert_eq!(added, 1, "host-only aliases are ambiguous when one host has multiple topology ports");
assert!(report.is_complete());
assert!(
servers
.iter()
.any(|server| server.disks.iter().any(|disk| disk.endpoint == "http://rustfs-1:9001/data1")),
"the second host:port member should be synthesized from exact topology"
);
}
#[test]
fn topology_report_detects_duplicate_drive_identity_with_balanced_total() {
let endpoints = topology_with_hosts(&["rustfs-1", "rustfs-2"]);
let servers = vec![server_with_disk("rustfs-1", 0, "ok"), server_with_disk("rustfs-1", 0, "ok")];
let report = server_topology_completeness_report(&servers, &endpoints);
assert_eq!(report.expected_drives, 2);
assert_eq!(report.observed_drives, 2, "a plain total-count check would look balanced");
assert_eq!(report.missing_drive_ids.len(), 1, "rustfs-2's drive identity is absent");
assert_eq!(report.duplicate_drive_ids.len(), 1, "rustfs-1's drive identity is duplicated");
assert!(!report.is_complete());
}
#[test]
fn data_usage_errors_are_sanitized_in_server_info() {
let mut buckets = rustfs_madmin::Buckets::default();
+2 -131
View File
@@ -22,10 +22,7 @@ use crate::diagnostics::get::{
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
use crate::erasure::coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
use bytes::Bytes;
use rustfs_config::{
DEFAULT_OBJECT_MMAP_READ_ENABLE, DEFAULT_OBJECT_MMAP_READ_MAX_LENGTH, ENV_OBJECT_MMAP_READ_ENABLE,
ENV_OBJECT_MMAP_READ_MAX_LENGTH, ENV_OBJECT_ZERO_COPY_ENABLE,
};
use rustfs_config::{DEFAULT_OBJECT_MMAP_READ_ENABLE, ENV_OBJECT_MMAP_READ_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE};
use rustfs_utils::HashAlgorithm;
use std::future::Future;
use std::io::{self, Cursor};
@@ -84,21 +81,6 @@ pub(crate) fn object_mmap_read_enabled() -> bool {
)
}
/// Mmap-copy read length cap. Cached: this is consulted on every shard open
/// and `std::env::var` takes a process-global lock. In test builds the env
/// var is read directly so `temp_env` overrides take effect.
pub(crate) fn object_mmap_read_max_length() -> usize {
#[cfg(test)]
{
rustfs_utils::get_env_usize(ENV_OBJECT_MMAP_READ_MAX_LENGTH, DEFAULT_OBJECT_MMAP_READ_MAX_LENGTH)
}
#[cfg(not(test))]
{
static CACHED: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
*CACHED.get_or_init(|| rustfs_utils::get_env_usize(ENV_OBJECT_MMAP_READ_MAX_LENGTH, DEFAULT_OBJECT_MMAP_READ_MAX_LENGTH))
}
}
#[derive(Clone)]
struct BitrotReaderSource {
inline_data: Option<Bytes>,
@@ -323,28 +305,7 @@ async fn open_disk_reader(
let metrics_path = metrics_path.filter(|_| rustfs_io_metrics::get_stage_metrics_enabled());
let stage_metrics_enabled = metrics_path.is_some();
// Mmap-copy materializes the whole `offset..offset+length` range as one
// owned allocation before any byte is served, and GET/heal shard reads
// request the entire part span in one call. Over-cap reads (e.g. a huge
// single-part object, rustfs#5123) take the bounded streaming path below.
let use_mmap_copy = if use_mmap_read && disk.is_local() {
let mmap_read_cap = object_mmap_read_max_length();
let within_cap = length <= mmap_read_cap;
if !within_cap {
rustfs_io_metrics::record_zero_copy_fallback("read_length_exceeds_cap");
debug!(
length,
mmap_read_cap,
path = %path,
"zero_copy_read_skipped_over_cap"
);
}
within_cap
} else {
false
};
if use_mmap_copy {
if use_mmap_read && disk.is_local() {
let start = stage_metrics_enabled.then(Instant::now);
let zero_copy_start = Instant::now();
let mmap_metrics = metrics_path.map(|metrics_path| MmapCopyStageMetrics {
@@ -724,96 +685,6 @@ mod tests {
);
}
#[test]
fn object_mmap_read_max_length_defaults_and_env_override() {
temp_env::with_var(ENV_OBJECT_MMAP_READ_MAX_LENGTH, None::<&str>, || {
assert_eq!(object_mmap_read_max_length(), DEFAULT_OBJECT_MMAP_READ_MAX_LENGTH);
});
temp_env::with_var(ENV_OBJECT_MMAP_READ_MAX_LENGTH, Some("1024"), || {
assert_eq!(object_mmap_read_max_length(), 1024);
});
temp_env::with_var(ENV_OBJECT_MMAP_READ_MAX_LENGTH, Some("0"), || {
assert_eq!(object_mmap_read_max_length(), 0);
});
}
// rustfs#5123: whole-part shard reads of large single-part objects must not
// be materialized in memory by the mmap-copy path; over-cap reads stream.
#[tokio::test]
async fn open_disk_reader_streams_when_length_exceeds_mmap_cap() {
use crate::disk::endpoint::Endpoint;
use crate::disk::{DiskOption, new_disk};
use tokio::io::AsyncReadExt;
let dir = tempfile::tempdir().expect("tempdir should be created");
let mut endpoint =
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("local disk should be created");
let payload = vec![7u8; 4096];
disk.make_volume("test-bucket").await.expect("volume should be created");
disk.write_all("test-bucket", "obj/part.1", Bytes::from(payload.clone()))
.await
.expect("shard file should be written");
temp_env::async_with_vars([(ENV_OBJECT_MMAP_READ_MAX_LENGTH, Some("1024"))], async {
let over_cap = open_disk_reader(&disk, "test-bucket", "obj/part.1", 0, payload.len(), true, None)
.await
.expect("over-cap read should open");
assert!(
matches!(over_cap, ShardReader::Stream(_)),
"read longer than the mmap cap must take the streaming path"
);
let mut over_cap = over_cap;
let mut streamed = Vec::new();
over_cap
.read_to_end(&mut streamed)
.await
.expect("streaming fallback should read the range");
assert_eq!(streamed, payload, "streaming fallback must return the same bytes");
let under_cap = open_disk_reader(&disk, "test-bucket", "obj/part.1", 0, 512, true, None)
.await
.expect("under-cap read should open");
assert!(
matches!(under_cap, ShardReader::InMemory(_)),
"read within the mmap cap keeps the mmap-copy fast path"
);
let at_cap = open_disk_reader(&disk, "test-bucket", "obj/part.1", 0, 1024, true, None)
.await
.expect("at-cap read should open");
assert!(
matches!(at_cap, ShardReader::InMemory(_)),
"read of exactly the mmap cap keeps the mmap-copy fast path"
);
})
.await;
// Cap of 0 disables mmap-copy for every non-empty read.
temp_env::async_with_vars([(ENV_OBJECT_MMAP_READ_MAX_LENGTH, Some("0"))], async {
let disabled = open_disk_reader(&disk, "test-bucket", "obj/part.1", 0, 512, true, None)
.await
.expect("read with cap 0 should open");
assert!(
matches!(disabled, ShardReader::Stream(_)),
"cap 0 must route every non-empty read to the streaming path"
);
})
.await;
}
#[tokio::test]
async fn test_create_bitrot_reader_with_inline_data() {
let test_data = b"hello world test data";
+11 -1
View File
@@ -406,6 +406,9 @@ impl WritePlan {
false,
)?,
WriteEncryptionMode::Singlepart { base_nonce } => HashReader::from_reader(
#[cfg(feature = "rio-v2")]
rustfs_rio::EncryptReader::new(reader, encryption.key_bytes, base_nonce),
#[cfg(not(feature = "rio-v2"))]
EncryptReader::new(reader, encryption.key_bytes, base_nonce),
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
@@ -417,6 +420,9 @@ impl WritePlan {
base_nonce,
multipart_part_number,
} => HashReader::from_reader(
#[cfg(feature = "rio-v2")]
rustfs_rio::EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number),
#[cfg(not(feature = "rio-v2"))]
EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number),
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
@@ -509,6 +515,10 @@ mod tests {
.await
.expect("read transformed ciphertext");
#[cfg(feature = "rio-v2")]
let decrypt_reader =
rustfs_rio::DecryptReader::new_multipart(Cursor::new(ciphertext), key_bytes, base_nonce, vec![part_number]);
#[cfg(not(feature = "rio-v2"))]
let decrypt_reader = DecryptReader::new_multipart(Cursor::new(ciphertext), key_bytes, base_nonce, vec![part_number]);
let mut decompressed = DecompressReader::new(Box::new(decrypt_reader), CompressionAlgorithm::default());
@@ -705,7 +715,7 @@ mod tests {
.expect("read transformed ciphertext");
let mut decrypted_compressed = Vec::new();
DecryptReader::new(Cursor::new(ciphertext), key_bytes, base_nonce)
rustfs_rio::DecryptReader::new(Cursor::new(ciphertext), key_bytes, base_nonce)
.read_to_end(&mut decrypted_compressed)
.await
.expect("decrypt compressed stream");
+9 -116
View File
@@ -1020,14 +1020,18 @@ impl EndpointServerPools {
#[instrument]
pub fn hosts_sorted(&self) -> Vec<Option<XHost>> {
let peers = self.peer_grid_hosts_sorted();
let (mut peers, local) = self.peers();
let mut ret = vec![None; peers.len()];
for (i, peer) in peers.into_iter().enumerate() {
let Some((peer_host_port, _)) = peer else {
peers.sort();
for (i, peer) in peers.iter().enumerate() {
if &local == peer {
continue;
};
let host = match XHost::try_from(peer_host_port) {
}
let host = match XHost::try_from(peer.clone()) {
Ok(res) => res,
Err(err) => {
warn!("Xhost parse failed {:?}", err);
@@ -1040,61 +1044,6 @@ impl EndpointServerPools {
ret
}
pub fn peer_host_ports_sorted(&self) -> Vec<Option<String>> {
let (mut peers, local) = self.peers();
let mut ret = vec![None; peers.len()];
peers.sort();
for (i, peer) in peers.into_iter().enumerate() {
if local == peer {
continue;
}
ret[i] = Some(peer);
}
ret
}
pub fn peer_grid_hosts_sorted(&self) -> Vec<Option<(String, String)>> {
self.peer_grid_host_slots_sorted()
.into_iter()
.map(|(peer, grid_host, is_local)| {
if is_local {
None
} else {
grid_host.map(|grid_host| (peer, grid_host))
}
})
.collect()
}
pub fn peer_grid_host_slots_sorted(&self) -> Vec<(String, Option<String>, bool)> {
let (mut peers, local) = self.peers();
let mut grid_hosts = HashMap::with_capacity(peers.len());
for ep in self.0.iter() {
for endpoint in ep.endpoints.0.iter() {
if endpoint.get_type() != EndpointType::Url || endpoint.is_local {
continue;
}
grid_hosts.entry(endpoint.host_port()).or_insert_with(|| endpoint.grid_host());
}
}
peers.sort();
peers
.into_iter()
.map(|peer| {
let is_local = local == peer;
let grid_host = if is_local { None } else { grid_hosts.get(&peer).cloned() };
(peer, grid_host, is_local)
})
.collect()
}
pub fn peers(&self) -> (Vec<String>, String) {
let mut local = None;
let mut set = HashSet::new();
@@ -1138,25 +1087,6 @@ impl EndpointServerPools {
None
}
pub fn find_grid_host_from_peer_host_port(&self, peer_host_port: &str) -> Option<String> {
for ep in self.0.iter() {
for endpoint in ep.endpoints.0.iter() {
if endpoint.is_local {
continue;
}
if endpoint.host_port() == peer_host_port {
return Some(endpoint.grid_host());
}
}
}
if let Ok(host) = XHost::try_from(peer_host_port.to_owned()) {
return self.find_grid_hosts_from_peer(&host);
}
None
}
}
fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()> {
@@ -1477,43 +1407,6 @@ mod test {
assert!(!eps[2].is_local, "a genuinely remote host must stay remote");
}
#[test]
fn peer_host_ports_sorted_preserves_raw_remote_hosts() {
let mut local =
Endpoint::try_from("http://rustfs-1.storage.swarm.private:9000/data").expect("local endpoint should parse");
local.is_local = true;
let mut remote =
Endpoint::try_from("http://rustfs-4.storage.swarm.private:9000/data").expect("remote endpoint should parse");
remote.is_local = false;
let endpoint_pools = EndpointServerPools::from(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 2,
endpoints: Endpoints::from(vec![local, remote]),
cmd_line: String::new(),
platform: String::new(),
}]);
let peers = endpoint_pools.peer_host_ports_sorted();
assert_eq!(
peers,
vec![None, Some("rustfs-4.storage.swarm.private:9000".to_string())],
"membership enumeration must keep raw topology host:port instead of requiring DNS resolution"
);
assert_eq!(
endpoint_pools
.find_grid_host_from_peer_host_port("rustfs-4.storage.swarm.private:9000")
.as_deref(),
Some("http://rustfs-4.storage.swarm.private:9000"),
"raw host:port should map directly to the configured grid host"
);
let peer_grid_hosts = endpoint_pools.peer_grid_hosts_sorted();
let remote = peer_grid_hosts[1].as_ref().expect("remote peer should be resolved");
assert_eq!(remote.0, "rustfs-4.storage.swarm.private:9000");
assert_eq!(remote.1, "http://rustfs-4.storage.swarm.private:9000");
}
#[tokio::test]
async fn orchestrated_policy_defers_dns_ip_same_path_check() {
// Two remote endpoints on the same address, same path, different ports
+1
View File
@@ -49,6 +49,7 @@ mod object_api;
mod runtime;
mod services;
mod set_disk;
mod sse;
mod storage_api_contracts;
mod store;
+350 -253
View File
@@ -14,6 +14,19 @@
use super::*;
#[cfg(feature = "rio-v2")]
use crate::sse::PersistedManagedEncryption;
#[cfg(feature = "rio-v2")]
use crate::sse::{
AMZ_SERVER_SIDE_ENCRYPTION_KMS_KEY_ID, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM, decrypt_minio_static_kms_dek,
};
use crate::sse::{
DEFAULT_SSE_ALGORITHM, INTERNAL_ENCRYPTION_IV_HEADER, INTERNAL_ENCRYPTION_KEY_HEADER, INTERNAL_ENCRYPTION_KEY_ID_HEADER,
MINIO_INTERNAL_ENCRYPTION_IV_HEADER, ManagedDekProvider, classify_persisted_managed_encryption,
};
#[cfg(feature = "rio-v2")]
use aes_gcm::aead::Payload;
use aes_gcm::{
Aes256Gcm, Key, Nonce,
@@ -26,46 +39,28 @@ use chacha20poly1305::ChaCha20Poly1305;
use hmac::{Hmac, Mac};
use md5::{Digest, Md5};
use rustfs_kms::types::ObjectEncryptionContext;
#[cfg(feature = "rio-v2")]
use rustfs_kms::{MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, decode_managed_kms_context};
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
use rustfs_utils::path::path_join_buf;
#[cfg(feature = "rio-v2")]
use serde::Deserialize;
#[cfg(feature = "rio-v2")]
use sha2::Sha256;
use std::collections::HashMap;
use std::env;
#[cfg(feature = "rio-v2")]
use zeroize::Zeroizing;
use crate::io_support::rio::Index;
const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id";
const INTERNAL_ENCRYPTION_KEY_HEADER: &str = "x-rustfs-encryption-key";
const INTERNAL_ENCRYPTION_CONTEXT_HEADER: &str = "x-rustfs-encryption-context";
const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv";
const INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER: &str = "x-rustfs-encryption-original-size";
const SSEC_ORIGINAL_SIZE_HEADER: &str = "x-amz-server-side-encryption-customer-original-size";
const DEFAULT_SSE_ALGORITHM: &str = "AES256";
#[cfg(feature = "rio-v2")]
const DARE_PAYLOAD_SIZE: i64 = 64 * 1024;
#[cfg(feature = "rio-v2")]
const DARE_PACKAGE_SIZE: i64 = DARE_PAYLOAD_SIZE + 32;
const MINIO_INTERNAL_ENCRYPTION_IV_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Iv";
#[cfg(feature = "rio-v2")]
const MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm";
#[cfg(feature = "rio-v2")]
const MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key";
#[cfg(feature = "rio-v2")]
const MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key";
#[cfg(feature = "rio-v2")]
const MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id";
#[cfg(feature = "rio-v2")]
const MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key";
#[cfg(feature = "rio-v2")]
const MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Context";
#[cfg(feature = "rio-v2")]
const MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key";
#[cfg(feature = "rio-v2")]
const MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM: &str = "DAREv2-HMAC-SHA256";
#[cfg(feature = "rio-v2")]
const DARE_VERSION_20: u8 = 0x20;
#[cfg(feature = "rio-v2")]
const DARE_CIPHER_AES_256_GCM: u8 = 0x00;
@@ -79,13 +74,6 @@ const DARE_TAG_SIZE: usize = 16;
const SEALED_KEY_IV_SIZE: usize = 32;
#[cfg(feature = "rio-v2")]
const SEALED_KEY_SIZE: usize = DARE_HEADER_SIZE + 32 + DARE_TAG_SIZE;
#[cfg(feature = "rio-v2")]
const MINIO_SECRET_KEY_RANDOM_SIZE: usize = 28;
#[cfg(feature = "rio-v2")]
const MINIO_SECRET_KEY_IV_SIZE: usize = 16;
#[cfg(feature = "rio-v2")]
const MINIO_SECRET_KEY_NONCE_SIZE: usize = 12;
#[cfg(feature = "rio-v2")]
type HmacSha256 = Hmac<Sha256>;
@@ -110,14 +98,6 @@ fn build_object_encryption_context(
object_context
}
#[cfg(feature = "rio-v2")]
fn is_legacy_rustfs_managed_metadata(metadata: &HashMap<String, String>) -> bool {
metadata_get(metadata, INTERNAL_ENCRYPTION_KEY_HEADER).is_some()
&& metadata_get(metadata, INTERNAL_ENCRYPTION_IV_HEADER).is_some()
&& metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER).is_none()
&& metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER).is_none()
}
fn part_plaintext_size(part: &ObjectPartInfo) -> i64 {
if part.actual_size > 0 {
part.actual_size
@@ -301,11 +281,19 @@ pub struct GetObjectReader {
pub stream: Box<dyn AsyncRead + Unpin + Send + Sync>,
pub object_info: ObjectInfo,
pub buffered_body: Option<Bytes>,
pub resolved_sse: Option<GetObjectSse>,
/// Cache-hook provenance; defaults to [`GetObjectBodySource::Unprobed`] for
/// every reader that never passed through the app-layer cache probe.
pub body_source: GetObjectBodySource,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum GetObjectSse {
SseC { customer_key_md5: String },
SseS3,
SseKms { key_id: String },
}
impl GetObjectReader {
/// Builds a fully materialized reader from a cache-coordinated body.
pub fn from_cache_body(mut object_info: ObjectInfo, body: Bytes) -> Result<Self> {
@@ -314,6 +302,7 @@ impl GetObjectReader {
stream: Box::new(std::io::Cursor::new(body.clone())),
object_info,
buffered_body: Some(body),
resolved_sse: None,
body_source: GetObjectBodySource::HookServed,
})
}
@@ -331,12 +320,13 @@ impl GetObjectReader {
}
}
#[derive(Debug, Clone, Copy)]
#[derive(Debug, Clone)]
struct EncryptionMaterial {
key_bytes: [u8; 32],
base_nonce: [u8; 12],
key_kind: EncryptionKeyKind,
reader_backend: crate::io_support::rio::ReadEncryptionBackend,
resolved_sse: Option<GetObjectSse>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -582,6 +572,7 @@ impl ReadPlan {
stream: reader,
object_info: oi.clone(),
buffered_body: None,
resolved_sse: None,
body_source: GetObjectBodySource::Unprobed,
},
self.storage_offset,
@@ -637,6 +628,7 @@ impl ReadPlan {
stream: final_reader,
object_info,
buffered_body: None,
resolved_sse: None,
body_source: GetObjectBodySource::Unprobed,
},
self.storage_offset,
@@ -733,12 +725,14 @@ impl ReadPlan {
let mut object_info = oi.clone();
object_info.size = self.object_size;
let resolved_sse = material.resolved_sse;
Ok((
GetObjectReader {
stream: final_reader,
object_info,
buffered_body: None,
resolved_sse,
body_source: GetObjectBodySource::Unprobed,
},
self.storage_offset,
@@ -1128,19 +1122,19 @@ fn is_supported_sealed_object_key_cipher(cipher: u8) -> bool {
}
#[cfg(feature = "rio-v2")]
fn decrypt_sealed_object_key_payload(sealing_key: [u8; 32], header: &[u8], sealed_key: &[u8]) -> Result<Vec<u8>> {
fn decrypt_sealed_object_key_payload(sealing_key: &[u8; 32], header: &[u8], sealed_key: &[u8]) -> Result<Vec<u8>> {
let nonce = &header[4..16];
let ciphertext = &sealed_key[DARE_HEADER_SIZE..];
let aad = &header[..4];
match header[1] {
DARE_CIPHER_AES_256_GCM => {
let cipher = Aes256Gcm::new_from_slice(&sealing_key)
let cipher = Aes256Gcm::new_from_slice(sealing_key)
.map_err(|err| Error::other(format!("invalid AES-GCM sealing key: {err}")))?;
let nonce = Nonce::try_from(nonce).map_err(|_| Error::other("invalid sealed object-key package nonce"))?;
cipher.decrypt(&nonce, Payload { msg: ciphertext, aad })
}
DARE_CIPHER_CHACHA20_POLY1305 => {
let cipher = ChaCha20Poly1305::new_from_slice(&sealing_key)
let cipher = ChaCha20Poly1305::new_from_slice(sealing_key)
.map_err(|err| Error::other(format!("invalid ChaCha20-Poly1305 sealing key: {err}")))?;
let nonce =
chacha20poly1305::Nonce::try_from(nonce).map_err(|_| Error::other("invalid sealed object-key package nonce"))?;
@@ -1287,8 +1281,8 @@ fn try_unseal_minio_object_key(
return Err(Error::other("invalid sealed object-key payload header"));
}
let sealing_key = derive_sealing_key(external_key, iv, managed_sse_domain(metadata), bucket, object);
let plaintext = decrypt_sealed_object_key_payload(sealing_key, header, &sealed_key)?;
let sealing_key = Zeroizing::new(derive_sealing_key(external_key, iv, managed_sse_domain(metadata), bucket, object));
let plaintext = Zeroizing::new(decrypt_sealed_object_key_payload(&sealing_key, header, &sealed_key)?);
let object_key: [u8; 32] = plaintext
.as_slice()
.try_into()
@@ -1336,12 +1330,19 @@ fn resolve_ssec_material(oi: &ObjectInfo, headers: &HeaderMap<HeaderValue>) -> R
}
#[cfg(feature = "rio-v2")]
if let Some(object_key) = try_unseal_minio_object_key(&oi.user_defined, &oi.bucket, &oi.name, key_bytes)? {
if metadata_get(&oi.user_defined, MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER).is_some()
|| metadata_get(&oi.user_defined, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER).is_some()
{
let object_key = try_unseal_minio_object_key(&oi.user_defined, &oi.bucket, &oi.name, key_bytes)?
.ok_or_else(|| Error::other("incomplete or invalid MinIO SSE-C sealed object-key metadata"))?;
return Ok(EncryptionMaterial {
key_bytes: object_key,
base_nonce: [0u8; 12],
key_kind: EncryptionKeyKind::Object,
reader_backend: crate::io_support::rio::ReadEncryptionBackend::V2,
resolved_sse: Some(GetObjectSse::SseC {
customer_key_md5: expected_md5,
}),
});
}
@@ -1350,6 +1351,9 @@ fn resolve_ssec_material(oi: &ObjectInfo, headers: &HeaderMap<HeaderValue>) -> R
base_nonce: read_stored_ssec_nonce(&oi.user_defined, &oi.bucket, &oi.name),
key_kind: EncryptionKeyKind::Direct,
reader_backend: crate::io_support::rio::ReadEncryptionBackend::Legacy,
resolved_sse: Some(GetObjectSse::SseC {
customer_key_md5: expected_md5,
}),
})
}
@@ -1372,28 +1376,65 @@ fn read_stored_ssec_nonce(metadata: &HashMap<String, String>, bucket: &str, key:
}
async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap<String, String>) -> Result<EncryptionMaterial> {
let normalized_metadata = normalize_managed_metadata(metadata);
let encrypted_dek = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_KEY_HEADER)
.ok_or_else(|| Error::other("missing managed encrypted DEK"))?;
let encrypted_dek = BASE64_STANDARD
.decode(encrypted_dek)
.map_err(|e| Error::other(format!("failed to decode managed encrypted DEK: {e}")))?;
let kms_key_id = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_KEY_ID_HEADER).unwrap_or("default");
let encrypted_dek = metadata_get(metadata, INTERNAL_ENCRYPTION_KEY_HEADER);
#[cfg(feature = "rio-v2")]
let kms_context = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_CONTEXT_HEADER)
.map(|value| {
serde_json::from_str::<HashMap<String, String>>(value)
.map_err(|e| Error::other(format!("failed to parse managed KMS context: {e}")))
})
.transpose()?;
let encrypted_dek = encrypted_dek.or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER));
let encrypted_dek = match encrypted_dek {
Some(encrypted_dek) => BASE64_STANDARD
.decode(encrypted_dek)
.map_err(|e| Error::other(format!("failed to decode managed encrypted DEK: {e}")))?,
None => Vec::new(),
};
let persisted_format =
classify_persisted_managed_encryption(metadata, &encrypted_dek).map_err(|err| Error::other(err.to_string()))?;
let kms_key_id = metadata_get(metadata, INTERNAL_ENCRYPTION_KEY_ID_HEADER).filter(|value| !value.is_empty());
#[cfg(feature = "rio-v2")]
let kms_key_id = kms_key_id
.or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER).filter(|value| !value.is_empty()))
.or_else(|| metadata_get(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_KEY_ID).filter(|value| !value.is_empty()));
let kms_key_id = kms_key_id.unwrap_or("default");
let resolved_sse = match persisted_format.scheme() {
crate::sse::ManagedSseScheme::SseS3 => GetObjectSse::SseS3,
crate::sse::ManagedSseScheme::SseKms => GetObjectSse::SseKms {
key_id: kms_key_id.to_string(),
},
};
#[cfg(feature = "rio-v2")]
let kms_context = decode_managed_kms_context(metadata).map_err(|err| Error::other(err.to_string()))?;
#[cfg(not(feature = "rio-v2"))]
let kms_context: Option<HashMap<String, String>> = None;
let object_context = build_object_encryption_context(bucket, object, kms_context.as_ref());
let decrypted_key = if let Some(service) = crate::runtime::sources::object_encryption_service().await {
let provider = persisted_format.provider();
#[cfg(feature = "rio-v2")]
let minio_static_key = if matches!(provider, ManagedDekProvider::Kms) {
decrypt_minio_static_kms_dek(kms_key_id, &encrypted_dek, &object_context.encryption_context)
.map_err(|err| Error::other(err.to_string()))?
} else {
None
};
#[cfg(not(feature = "rio-v2"))]
let minio_static_key: Option<[u8; 32]> = None;
let service = match provider {
ManagedDekProvider::LocalSseS3 | ManagedDekProvider::MinioKeyValue => None,
ManagedDekProvider::Kms if minio_static_key.is_some() => None,
ManagedDekProvider::Kms => Some(
crate::runtime::sources::object_encryption_service()
.await
.ok_or_else(|| Error::other("KMS encryption service is required to decrypt this object"))?,
),
};
let decrypted_key = if let Some(key) = minio_static_key {
key
} else if let Some(service) = service {
#[cfg(feature = "rio-v2")]
let data_key = if is_legacy_rustfs_managed_metadata(&normalized_metadata) {
let data_key = if matches!(
persisted_format,
PersistedManagedEncryption::LegacySseS3Local
| PersistedManagedEncryption::LegacySseS3Kms
| PersistedManagedEncryption::LegacySseKms
) {
service.decrypt_legacy_data_key(&encrypted_dek).await
} else {
service.decrypt_data_key(&encrypted_dek, &object_context).await
@@ -1405,20 +1446,24 @@ async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap
.map_err(|e| Error::other(format!("failed to decrypt managed data key: {e}")))?
.plaintext_key
} else {
decrypt_local_sse_dek(&encrypted_dek, kms_key_id, &object_context)?
decrypt_local_sse_dek(&encrypted_dek)?
};
#[cfg(feature = "rio-v2")]
if let Some(object_key) = try_unseal_minio_object_key(&normalized_metadata, bucket, object, decrypted_key)? {
if persisted_format.uses_object_key() {
let object_key = try_unseal_minio_object_key(metadata, bucket, object, decrypted_key)?
.ok_or_else(|| Error::other("MinIO managed SSE metadata is missing a valid sealed object key"))?;
return Ok(EncryptionMaterial {
key_bytes: object_key,
base_nonce: [0u8; 12],
key_kind: EncryptionKeyKind::Object,
reader_backend: crate::io_support::rio::ReadEncryptionBackend::V2,
resolved_sse: Some(resolved_sse),
});
}
let iv_b64 = metadata_get(&normalized_metadata, INTERNAL_ENCRYPTION_IV_HEADER)
let iv_b64 = metadata_get(metadata, INTERNAL_ENCRYPTION_IV_HEADER)
.or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER))
.ok_or_else(|| Error::other("missing managed encryption IV"))?;
let iv = BASE64_STANDARD
.decode(iv_b64)
@@ -1433,66 +1478,15 @@ async fn resolve_managed_material(bucket: &str, object: &str, metadata: &HashMap
base_nonce,
key_kind: EncryptionKeyKind::Direct,
reader_backend: crate::io_support::rio::ReadEncryptionBackend::Legacy,
resolved_sse: Some(resolved_sse),
})
}
fn normalize_managed_metadata(metadata: &HashMap<String, String>) -> HashMap<String, String> {
#[cfg(feature = "rio-v2")]
{
let mut normalized = metadata.clone();
if metadata_get(&normalized, INTERNAL_ENCRYPTION_KEY_HEADER).is_none()
&& let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER)
.or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER))
.or_else(|| metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER))
{
normalized.insert(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), value.to_string());
}
if metadata_get(&normalized, INTERNAL_ENCRYPTION_IV_HEADER).is_none()
&& let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER)
{
normalized.insert(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), value.to_string());
}
if metadata_get(&normalized, INTERNAL_ENCRYPTION_KEY_ID_HEADER).is_none()
&& let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER)
{
normalized.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), value.to_string());
}
if metadata_get(&normalized, INTERNAL_ENCRYPTION_CONTEXT_HEADER).is_none()
&& let Some(value) = metadata_get(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)
&& let Ok(decoded) = BASE64_STANDARD.decode(value)
&& let Ok(context) = serde_json::from_slice::<HashMap<String, String>>(&decoded)
&& let Ok(encoded) = serde_json::to_string(&context)
{
normalized.insert(INTERNAL_ENCRYPTION_CONTEXT_HEADER.to_string(), encoded);
}
normalized
}
#[cfg(not(feature = "rio-v2"))]
{
metadata.clone()
}
}
fn decrypt_local_sse_dek(encrypted_dek: &[u8], _kms_key_id: &str, object_context: &ObjectEncryptionContext) -> Result<[u8; 32]> {
if let Ok(plaintext) = decrypt_rustfs_local_sse_dek(encrypted_dek) {
return Ok(plaintext);
}
#[cfg(feature = "rio-v2")]
{
decrypt_minio_secret_key_dek(encrypted_dek, object_context)
}
#[cfg(not(feature = "rio-v2"))]
{
let _ = object_context;
Err(Error::other("invalid managed DEK format"))
fn decrypt_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> {
if encrypted_dek.is_empty() {
return local_sse_master_key();
}
decrypt_rustfs_local_sse_dek(encrypted_dek)
}
fn decrypt_rustfs_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> {
@@ -1526,113 +1520,22 @@ fn decrypt_rustfs_local_sse_dek(encrypted_dek: &[u8]) -> Result<[u8; 32]> {
.map_err(|_| Error::other("managed DEK has invalid plaintext length"))
}
#[cfg(feature = "rio-v2")]
#[derive(Deserialize)]
struct MinioLegacyCiphertext {
#[serde(rename = "aead")]
algorithm: String,
iv: Vec<u8>,
nonce: Vec<u8>,
bytes: Vec<u8>,
}
#[cfg(feature = "rio-v2")]
fn decrypt_minio_secret_key_dek(encrypted_dek: &[u8], object_context: &ObjectEncryptionContext) -> Result<[u8; 32]> {
let key = local_sse_master_key()?;
let (ciphertext, iv, nonce) = parse_minio_secret_key_ciphertext(encrypted_dek)?;
let associated_data = marshal_minio_kms_context(&object_context.encryption_context);
let mut mac = HmacSha256::new_from_slice(&key).map_err(|err| Error::other(format!("invalid local SSE master key: {err}")))?;
mac.update(&iv);
let sealing_key = mac.finalize().into_bytes();
let cipher = Aes256Gcm::new_from_slice(sealing_key.as_slice())
.map_err(|err| Error::other(format!("invalid MinIO sealing key: {err}")))?;
let nonce = Nonce::try_from(&nonce[..]).map_err(|_| Error::other("invalid MinIO managed DEK nonce"))?;
let plaintext = cipher
.decrypt(
&nonce,
aes_gcm::aead::Payload {
msg: &ciphertext,
aad: &associated_data,
},
)
.map_err(|err| Error::other(format!("failed to decrypt MinIO managed DEK: {err}")))?;
plaintext
.as_slice()
.try_into()
.map_err(|_| Error::other("MinIO managed DEK has invalid plaintext length"))
}
#[cfg(feature = "rio-v2")]
fn parse_minio_secret_key_ciphertext(
encrypted_dek: &[u8],
) -> Result<(Vec<u8>, [u8; MINIO_SECRET_KEY_IV_SIZE], [u8; MINIO_SECRET_KEY_NONCE_SIZE])> {
if encrypted_dek.first() == Some(&b'{') && encrypted_dek.last() == Some(&b'}') {
let legacy: MinioLegacyCiphertext = serde_json::from_slice(encrypted_dek)
.map_err(|err| Error::other(format!("failed to parse MinIO legacy managed DEK: {err}")))?;
if legacy.algorithm != "AES-256-GCM-HMAC-SHA-256" {
return Err(Error::other(format!(
"unsupported MinIO legacy managed DEK algorithm {}",
legacy.algorithm
)));
}
let iv = legacy
.iv
.as_slice()
.try_into()
.map_err(|_| Error::other("invalid MinIO legacy managed DEK IV length"))?;
let nonce = legacy
.nonce
.as_slice()
.try_into()
.map_err(|_| Error::other("invalid MinIO legacy managed DEK nonce length"))?;
return Ok((legacy.bytes, iv, nonce));
}
if encrypted_dek.len() <= MINIO_SECRET_KEY_RANDOM_SIZE {
return Err(Error::other("invalid MinIO managed DEK length"));
}
let split_at = encrypted_dek.len() - MINIO_SECRET_KEY_RANDOM_SIZE;
let (ciphertext, random) = encrypted_dek.split_at(split_at);
let iv = random[..MINIO_SECRET_KEY_IV_SIZE]
.try_into()
.map_err(|_| Error::other("invalid MinIO managed DEK IV length"))?;
let nonce = random[MINIO_SECRET_KEY_IV_SIZE..]
.try_into()
.map_err(|_| Error::other("invalid MinIO managed DEK nonce length"))?;
Ok((ciphertext.to_vec(), iv, nonce))
}
#[cfg(feature = "rio-v2")]
fn marshal_minio_kms_context(context: &HashMap<String, String>) -> Vec<u8> {
let mut entries: Vec<_> = context.iter().collect();
entries.sort_by_key(|(left, _)| *left);
let mut json = String::from("{");
for (index, (key, value)) in entries.into_iter().enumerate() {
if index > 0 {
json.push(',');
}
json.push_str(&serde_json::to_string(key).expect("string key serializes"));
json.push(':');
json.push_str(&serde_json::to_string(value).expect("string value serializes"));
}
json.push('}');
json.into_bytes()
}
fn local_sse_master_key() -> Result<[u8; 32]> {
#[cfg(test)]
if let Some(key) = decode_master_key_env("__RUSTFS_SSE_SIMPLE_CMK")? {
return Ok(key);
}
if let Some(key) = decode_master_key_env("RUSTFS_SSE_S3_MASTER_KEY")? {
if key == [0u8; 32] {
return Err(Error::other("RUSTFS_SSE_S3_MASTER_KEY must not be an all-zero key"));
}
return Ok(key);
}
Ok([0u8; 32])
Err(Error::other(
"SSE-S3 requires RUSTFS_SSE_S3_MASTER_KEY to decrypt locally managed objects",
))
}
fn decode_master_key_env(name: &str) -> Result<Option<[u8; 32]>> {
@@ -1707,6 +1610,7 @@ mod tests {
assert_eq!(reader.body_source, GetObjectBodySource::HookServed);
assert_eq!(reader.buffered_body.as_ref(), Some(&body));
assert_eq!(reader.resolved_sse, None);
assert_eq!(reader.object_info.size, 11);
assert_eq!(reader.object_info.actual_size, 11);
assert!(reader.object_info.is_compressed());
@@ -1767,20 +1671,38 @@ mod tests {
#[cfg(feature = "rio-v2")]
#[test]
fn test_legacy_managed_metadata_excludes_sealed_keys() {
let legacy_metadata = HashMap::from([
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), "encrypted-dek".to_string()),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), "nonce".to_string()),
]);
assert!(is_legacy_rustfs_managed_metadata(&legacy_metadata));
fn resolve_ssec_material_rejects_partial_object_key_metadata() {
let customer_key = [0x42u8; 32];
let headers = ssec_headers_from_key(customer_key);
for partial in [
HashMap::from([(
MINIO_INTERNAL_ENCRYPTION_SSEC_SEALED_KEY_HEADER.to_string(),
BASE64_STANDARD.encode([0x11u8; SEALED_KEY_SIZE]),
)]),
HashMap::from([(
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
)]),
] {
let mut metadata = HashMap::from([
(SSEC_ALGORITHM_HEADER.to_string(), DEFAULT_SSE_ALGORITHM.to_string()),
(SSEC_KEY_MD5_HEADER.to_string(), BASE64_STANDARD.encode(md5_bytes(customer_key))),
]);
metadata.extend(partial);
let object_info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
user_defined: Arc::new(metadata),
..Default::default()
};
let sealed_metadata = HashMap::from([
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), "encrypted-dek".to_string()),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), "nonce".to_string()),
(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(), "sealed-key".to_string()),
]);
assert!(!is_legacy_rustfs_managed_metadata(&sealed_metadata));
assert!(
resolve_ssec_material(&object_info, &headers)
.expect_err("partial MinIO SSE-C metadata must fail closed")
.to_string()
.contains("incomplete or invalid")
);
}
}
#[cfg(feature = "rio-v2")]
@@ -2125,9 +2047,21 @@ mod tests {
data_key: [u8; 32],
object_key: [u8; 32],
cipher_id: u8,
) -> ([u8; 32], Vec<u8>) {
seal_managed_object_key_for_test(bucket, object, data_key, object_key, "SSE-S3", cipher_id)
}
#[cfg(feature = "rio-v2")]
fn seal_managed_object_key_for_test(
bucket: &str,
object: &str,
data_key: [u8; 32],
object_key: [u8; 32],
domain: &str,
cipher_id: u8,
) -> ([u8; 32], Vec<u8>) {
let iv = [0x24u8; SEALED_KEY_IV_SIZE];
let sealing_key = derive_sealing_key(data_key, iv, "SSE-S3", bucket, object);
let sealing_key = derive_sealing_key(data_key, iv, domain, bucket, object);
let mut header = [0u8; DARE_HEADER_SIZE];
header[0] = DARE_VERSION_20;
@@ -2203,7 +2137,7 @@ mod tests {
#[tokio::test]
async fn resolve_managed_material_accepts_chacha20_poly1305_header_variant() {
async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async {
let data_key = [0x24; 32];
let data_key = [0u8; 32];
let object_key = [0x33; 32];
let (iv, sealed_key) = seal_managed_s3_object_key_for_test_with_cipher(
"bucket",
@@ -2213,7 +2147,6 @@ mod tests {
DARE_CIPHER_CHACHA20_POLY1305,
);
let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]);
let metadata = HashMap::from([
(
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(),
@@ -2224,11 +2157,6 @@ mod tests {
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
),
(
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(encrypted_dek.as_bytes()),
),
(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()),
]);
let material = resolve_managed_material("bucket", "object", &metadata)
@@ -2458,11 +2386,52 @@ mod tests {
));
}
#[tokio::test]
async fn test_zero_length_ssec_read_still_requires_customer_key() {
let object_info = ObjectInfo {
size: 0,
user_defined: Arc::new(HashMap::from([
(SSEC_ALGORITHM_HEADER.to_string(), DEFAULT_SSE_ALGORITHM.to_string()),
(SSEC_KEY_MD5_HEADER.to_string(), BASE64_STANDARD.encode([0x11; 16])),
(SSEC_ORIGINAL_SIZE_HEADER.to_string(), "0".to_string()),
])),
..Default::default()
};
let result = ReadPlan::build(None, &object_info, &ObjectOptions::default(), &HeaderMap::new()).await;
assert!(result.is_err(), "zero-length SSE-C reads must not bypass customer-key authorization");
}
#[tokio::test]
async fn test_zero_length_sse_kms_read_still_requires_kms() {
let object_info = ObjectInfo {
size: 0,
bucket: "bucket".to_string(),
name: "zero-kms".to_string(),
user_defined: Arc::new(HashMap::from([
("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()),
(
INTERNAL_ENCRYPTION_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(b"opaque-kms-ciphertext"),
),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([0x12; 12])),
(INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(), "0".to_string()),
])),
..Default::default()
};
let result = ReadPlan::build(None, &object_info, &ObjectOptions::default(), &HeaderMap::new()).await;
assert!(result.is_err(), "zero-length SSE-KMS reads must not bypass KMS authorization");
}
#[tokio::test]
async fn test_get_object_reader_allows_encrypted_full_object_passthrough() {
async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async {
let plaintext = b"managed-full-object".to_vec();
let data_key = [0x21; 32];
let data_key = [0u8; 32];
#[cfg(not(feature = "rio-v2"))]
let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]);
let bucket = "bucket";
let object = "managed-full-object";
@@ -2478,7 +2447,6 @@ mod tests {
.expect("encrypt managed object");
HashMap::from([
("x-amz-server-side-encryption".to_string(), "AES256".to_string()),
("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())),
("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()),
(
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
@@ -2539,7 +2507,8 @@ mod tests {
async fn test_get_object_reader_decrypts_managed_sse_range_on_plaintext_semantics() {
async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async {
let plaintext = b"0123456789abcdefghijklmnopqrstuvwxyz".to_vec();
let data_key = [0x23; 32];
let data_key = [0u8; 32];
#[cfg(not(feature = "rio-v2"))]
let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]);
let bucket = "bucket";
let object = "managed-range-object";
@@ -2555,7 +2524,6 @@ mod tests {
.expect("encrypt managed ranged object");
HashMap::from([
("x-amz-server-side-encryption".to_string(), "AES256".to_string()),
("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())),
("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()),
(
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
@@ -2622,12 +2590,13 @@ mod tests {
async_with_vars(
[
("__RUSTFS_SSE_SIMPLE_CMK", None::<String>),
("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode([0u8; 32]))),
("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode([0x33u8; 32]))),
],
async {
let plaintext = b"managed-local-fallback".to_vec();
let data_key = [0x22; 32];
let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]);
let data_key = [0x33; 32];
#[cfg(not(feature = "rio-v2"))]
let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0x33; 32]);
let bucket = "bucket";
let object = "managed-local-fallback";
@@ -2642,7 +2611,6 @@ mod tests {
.expect("encrypt managed object with local fallback key");
HashMap::from([
("x-amz-server-side-encryption".to_string(), "AES256".to_string()),
("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(encrypted_dek.as_bytes())),
("x-rustfs-encryption-original-size".to_string(), plaintext.len().to_string()),
(
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
@@ -2703,8 +2671,7 @@ mod tests {
async fn test_get_object_reader_accepts_minio_only_managed_metadata() {
async_with_vars([("__RUSTFS_SSE_SIMPLE_CMK", Some(BASE64_STANDARD.encode([0u8; 32])))], async {
let plaintext = b"managed-minio-metadata".to_vec();
let data_key = [0x23; 32];
let encrypted_dek = encrypt_managed_dek_for_test(data_key, [0u8; 32]);
let data_key = [0u8; 32];
let bucket = "bucket";
let object = "managed-minio-metadata";
let object_key = [0x44; 32];
@@ -2722,10 +2689,6 @@ mod tests {
size: encrypted.len() as i64,
user_defined: Arc::new(HashMap::from([
("x-amz-server-side-encryption".to_string(), "AES256".to_string()),
(
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(encrypted_dek.as_bytes()),
),
(
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(sealed_key),
@@ -2735,7 +2698,6 @@ mod tests {
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
),
(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()),
("x-minio-internal-actual-size".to_string(), plaintext.len().to_string()),
])),
..Default::default()
@@ -2751,6 +2713,7 @@ mod tests {
.await
.expect("managed encrypted reads should accept MinIO-style metadata");
assert_eq!(reader.resolved_sse, Some(GetObjectSse::SseS3));
let mut actual = Vec::new();
reader.read_to_end(&mut actual).await.expect("read managed plaintext");
@@ -2762,6 +2725,128 @@ mod tests {
.await;
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn test_get_object_reader_accepts_minio_legacy_key_value_metadata() {
let external_key = [0x23; 32];
async_with_vars([("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode(external_key)))], async {
let plaintext = b"managed-minio-key-value-metadata".to_vec();
let bucket = "bucket";
let object = "managed-minio-key-value-metadata";
let object_key = [0x45; 32];
let (sealing_iv, sealed_key) = seal_managed_s3_object_key_for_test(bucket, object, external_key, object_key);
let mut encrypted = Vec::new();
crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key)
.read_to_end(&mut encrypted)
.await
.expect("encrypt managed object");
let object_info = ObjectInfo {
bucket: bucket.to_string(),
name: object.to_string(),
size: encrypted.len() as i64,
user_defined: Arc::new(HashMap::from([
("x-amz-server-side-encryption".to_string(), "AES256".to_string()),
(
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(sealed_key),
),
(MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)),
(
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
),
("x-minio-internal-actual-size".to_string(), plaintext.len().to_string()),
])),
..Default::default()
};
let (mut reader, _, _) = GetObjectReader::new(
Box::new(Cursor::new(encrypted)),
None,
&object_info,
&ObjectOptions::default(),
&HeaderMap::new(),
)
.await
.expect("MinIO historical K/V metadata should remain readable");
let mut actual = Vec::new();
reader
.read_to_end(&mut actual)
.await
.expect("read MinIO historical K/V object");
assert_eq!(reader.resolved_sse, Some(GetObjectSse::SseS3));
assert_eq!(actual, plaintext);
})
.await;
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn test_get_object_reader_accepts_minio_legacy_kms_key_value_metadata() {
let external_key = [0x26; 32];
async_with_vars([("RUSTFS_SSE_S3_MASTER_KEY", Some(BASE64_STANDARD.encode(external_key)))], async {
let plaintext = b"managed-minio-kms-key-value-metadata".to_vec();
let bucket = "bucket";
let object = "managed-minio-kms-key-value-metadata";
let object_key = [0x48; 32];
let (sealing_iv, sealed_key) =
seal_managed_object_key_for_test(bucket, object, external_key, object_key, "SSE-KMS", DARE_CIPHER_AES_256_GCM);
let mut encrypted = Vec::new();
crate::io_support::rio::EncryptReader::new_with_object_key(Cursor::new(plaintext.clone()), object_key)
.read_to_end(&mut encrypted)
.await
.expect("encrypt managed KMS object");
let object_info = ObjectInfo {
bucket: bucket.to_string(),
name: object.to_string(),
size: encrypted.len() as i64,
user_defined: Arc::new(HashMap::from([
("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()),
(
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(sealed_key),
),
(MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode(sealing_iv)),
(
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
),
("x-minio-internal-actual-size".to_string(), plaintext.len().to_string()),
])),
..Default::default()
};
let (mut reader, _, _) = GetObjectReader::new(
Box::new(Cursor::new(encrypted)),
None,
&object_info,
&ObjectOptions::default(),
&HeaderMap::new(),
)
.await
.expect("MinIO historical SSE-KMS K/V metadata should remain readable");
let mut actual = Vec::new();
reader
.read_to_end(&mut actual)
.await
.expect("read MinIO historical SSE-KMS K/V object");
assert_eq!(
reader.resolved_sse,
Some(GetObjectSse::SseKms {
key_id: "default".to_string(),
})
);
assert_eq!(actual, plaintext);
})
.await;
}
#[tokio::test]
async fn test_get_object_reader_compressed_range_returns_physical_offset_from_index() {
let mut index = Index::new();
@@ -3034,6 +3119,12 @@ mod tests {
.await
.expect("ssec read should be supported");
assert_eq!(
reader.resolved_sse,
Some(GetObjectSse::SseC {
customer_key_md5: BASE64_STANDARD.encode(md5_bytes(key_bytes)),
})
);
let mut actual = Vec::new();
reader.read_to_end(&mut actual).await.expect("read decrypted ssec object");
@@ -3096,6 +3187,12 @@ mod tests {
.await
.expect("rio-v2 ssec sealed-object-key read should be supported");
assert_eq!(
reader.resolved_sse,
Some(GetObjectSse::SseC {
customer_key_md5: BASE64_STANDARD.encode(md5_bytes(customer_key)),
})
);
let mut actual = Vec::new();
reader
.read_to_end(&mut actual)
+22 -164
View File
@@ -95,18 +95,16 @@ pub fn get_global_notification_sys() -> Option<Arc<NotificationSys>> {
pub struct NotificationSys {
pub peer_clients: Vec<Option<PeerRestClient>>,
pub all_peer_clients: Vec<Option<PeerRestClient>>,
peer_topology_hosts: Vec<String>,
peer_admin_caches: Vec<Mutex<PeerAdminCache>>,
}
impl NotificationSys {
pub async fn new(eps: EndpointServerPools) -> Self {
let (peer_clients, all_peer_clients, peer_topology_hosts) = PeerRestClient::new_clients_with_topology(eps).await;
let (peer_clients, all_peer_clients) = PeerRestClient::new_clients(eps).await;
let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
Self {
peer_clients,
all_peer_clients,
peer_topology_hosts,
peer_admin_caches,
}
}
@@ -381,19 +379,22 @@ impl NotificationSys {
let peer_timeout = Duration::from_secs(5);
for (idx, client) in self.peer_clients.iter().enumerate() {
let host = self
.peer_topology_hosts
.get(idx)
.cloned()
.or_else(|| client.as_ref().map(|client| client.host.to_string()))
.unwrap_or_default();
let endpoints = endpoints.clone();
let cache = self.peer_admin_caches.get(idx);
futures.push(async move {
// `peer_clients` comes from `new_clients`, which only ever pushes
// `Some(client)` (local hosts are excluded, not slotted as
// `None`), so this branch is unreachable in practice. Kept as a
// defensive fallback: report an explicit `unknown` state rather
// than a blank `default()` entry, so it can never contribute a
// hollow row to `servers[]` (rustfs/backlog#1049 P3).
let Some(client) = client else {
return PeerServerInfoProbe {
host,
result: Err(PeerServerInfoProbeFailure::NoClient),
return ServerProperties {
state: ItemState::Unknown.to_string().to_owned(),
..Default::default()
};
};
let host = client.host.to_string();
// First attempt. A single evicted or half-open internode channel
// is enough to fail one probe and, before retrying, would drop
@@ -402,7 +403,8 @@ impl NotificationSys {
// before falling back (rustfs/backlog#1049, P1-B).
match timeout(peer_timeout, client.server_info()).await {
Ok(Ok(info)) => {
return PeerServerInfoProbe { host, result: Ok(info) };
update_server_info_cache(cache, &host, &info);
return info;
}
Ok(Err(err)) => debug!("peer {host} server_info failed (attempt 1/2): {err}"),
Err(_) => debug!("peer {host} server_info timed out (attempt 1/2) after {peer_timeout:?}"),
@@ -418,29 +420,26 @@ impl NotificationSys {
// Second and final attempt on the fresh channel.
match timeout(peer_timeout, client.server_info()).await {
Ok(Ok(info)) => PeerServerInfoProbe { host, result: Ok(info) },
Ok(Ok(info)) => {
update_server_info_cache(cache, &host, &info);
info
}
Ok(Err(err)) => {
warn!("peer {host} server_info failed after retry: {err}");
let health = peer_disk_health(&host).await;
PeerServerInfoProbe {
host,
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
}
handle_server_info_failure(cache, &host, &endpoints, health.as_ref())
}
Err(_) => {
warn!("peer {host} server_info timed out after retry ({peer_timeout:?})");
client.evict_connection().await;
let health = peer_disk_health(&host).await;
PeerServerInfoProbe {
host,
result: Err(PeerServerInfoProbeFailure::Rpc { health }),
}
handle_server_info_failure(cache, &host, &endpoints, health.as_ref())
}
}
});
}
publish_server_info_probe_round(&self.peer_admin_caches, &endpoints, join_all(futures).await)
join_all(futures).await
}
pub async fn load_user(&self, access_key: &str, temp: bool) -> Vec<NotificationPeerErr> {
@@ -1264,16 +1263,6 @@ struct PeerDiskHealth {
disks: Vec<rustfs_madmin::Disk>,
}
struct PeerServerInfoProbe {
host: String,
result: std::result::Result<ServerProperties, PeerServerInfoProbeFailure>,
}
enum PeerServerInfoProbeFailure {
Rpc { health: Option<PeerDiskHealth> },
NoClient,
}
/// Consult the local disk-health state for `host` without issuing any RPC.
///
/// On the aggregating node a peer's drives are remote-disk handles whose
@@ -1435,30 +1424,6 @@ fn handle_server_info_failure(
unknown_server_properties(host, endpoints)
}
fn publish_server_info_probe_round(
caches: &[Mutex<PeerAdminCache>],
endpoints: &EndpointServerPools,
probes: Vec<PeerServerInfoProbe>,
) -> Vec<ServerProperties> {
probes
.into_iter()
.enumerate()
.map(|(idx, probe)| {
let cache = caches.get(idx);
match probe.result {
Ok(info) => {
update_server_info_cache(cache, &probe.host, &info);
info
}
Err(PeerServerInfoProbeFailure::Rpc { health }) => {
handle_server_info_failure(cache, &probe.host, endpoints, health.as_ref())
}
Err(PeerServerInfoProbeFailure::NoClient) => unknown_server_properties(&probe.host, endpoints),
}
})
.collect()
}
fn update_server_info_cache(cache: Option<&Mutex<PeerAdminCache>>, host: &str, info: &ServerProperties) {
let Some(cache) = cache else {
return;
@@ -1669,7 +1634,6 @@ mod tests {
"127.0.0.1:9000".to_string().try_into().expect("peer host should parse"),
"http://127.0.0.1:9000".to_string(),
))],
peer_topology_hosts: Vec::new(),
peer_admin_caches: Vec::new(),
};
@@ -1713,7 +1677,6 @@ mod tests {
let sys = NotificationSys {
peer_clients: vec![None],
all_peer_clients: Vec::new(),
peer_topology_hosts: vec!["node-a:9000".to_string()],
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
};
@@ -1733,7 +1696,6 @@ mod tests {
let sys = NotificationSys {
peer_clients: vec![None],
all_peer_clients: vec![None, None],
peer_topology_hosts: vec!["node-a:9000".to_string()],
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
};
@@ -1750,7 +1712,6 @@ mod tests {
let sys = NotificationSys {
peer_clients: Vec::new(),
all_peer_clients: Vec::new(),
peer_topology_hosts: Vec::new(),
peer_admin_caches: Vec::new(),
};
@@ -1771,7 +1732,6 @@ mod tests {
let sys = NotificationSys {
peer_clients: vec![Some(client)],
all_peer_clients: vec![None],
peer_topology_hosts: vec!["127.0.0.1:9000".to_string()],
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
};
@@ -1783,51 +1743,6 @@ mod tests {
assert!(err.to_string().contains("peer topology is incomplete"));
}
#[tokio::test]
async fn server_info_no_client_slot_uses_topology_host_without_counting_rpc_failure() {
let sys = NotificationSys {
peer_clients: vec![None],
all_peer_clients: vec![None, None],
peer_topology_hosts: vec!["node-a:9000".to_string()],
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
};
let servers = sys.server_info().await;
assert_eq!(servers.len(), 1);
assert_eq!(servers[0].endpoint, "node-a:9000");
assert_eq!(servers[0].state, ItemState::Unknown.to_string());
let cache = sys.peer_admin_caches[0].lock().expect("cache mutex should not be poisoned");
assert_eq!(cache.server_failures, 0, "construction-only missing slots are not failed RPC attempts");
assert!(cache.last_server_info.is_none());
}
#[test]
fn server_info_failure_cache_stays_aligned_with_topology_slot() {
let cache_a = Mutex::new(PeerAdminCache {
last_server_info: Some(build_props("cached-a")),
last_server_success: Some(SystemTime::now()),
server_failures: 1,
storage_failures: 0,
last_storage_info: None,
});
let cache_b = Mutex::new(PeerAdminCache {
last_server_info: Some(build_props("cached-b")),
last_server_success: Some(SystemTime::now()),
server_failures: 1,
storage_failures: 0,
last_storage_info: None,
});
let caches = [cache_a, cache_b];
let endpoints = EndpointServerPools::from(Vec::new());
let rendered = handle_server_info_failure(Some(&caches[1]), "node-b:9000", &endpoints, None);
assert_eq!(rendered.endpoint, "cached-b");
assert_eq!(caches[0].lock().expect("cache mutex should not be poisoned").server_failures, 1);
assert_eq!(caches[1].lock().expect("cache mutex should not be poisoned").server_failures, 2);
}
#[tokio::test]
async fn scanner_activity_probe_times_out() {
let err = scanner_activity_with_timeout(
@@ -1847,7 +1762,6 @@ mod tests {
let sys = NotificationSys {
peer_clients: vec![None],
all_peer_clients: Vec::new(),
peer_topology_hosts: vec!["node-a:9000".to_string()],
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
};
@@ -1867,7 +1781,6 @@ mod tests {
let sys = NotificationSys {
peer_clients: vec![None],
all_peer_clients: Vec::new(),
peer_topology_hosts: vec!["node-a:9000".to_string()],
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
};
@@ -1883,7 +1796,6 @@ mod tests {
let sys = NotificationSys {
peer_clients: vec![None],
all_peer_clients: Vec::new(),
peer_topology_hosts: vec!["node-a:9000".to_string()],
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
};
let mutation_id = Uuid::from_u128(1);
@@ -2084,60 +1996,6 @@ mod tests {
assert!(!cached_snapshot_is_fresh(Some(stale)), "an old success is stale");
}
#[test]
fn server_info_probe_round_commits_failures_only_when_published() {
let caches = vec![Mutex::new(PeerAdminCache::new())];
let endpoints = EndpointServerPools::default();
let probes = vec![PeerServerInfoProbe {
host: "peer-1".to_string(),
result: Err(PeerServerInfoProbeFailure::Rpc { health: None }),
}];
assert_eq!(
caches[0]
.lock()
.expect("peer cache should lock before publish")
.server_failures,
0
);
let replies = publish_server_info_probe_round(&caches, &endpoints, probes);
assert_eq!(replies.len(), 1);
assert_eq!(replies[0].endpoint, "peer-1");
assert_eq!(replies[0].state, ItemState::Unknown.to_string());
assert_eq!(
caches[0]
.lock()
.expect("peer cache should lock after publish")
.server_failures,
1
);
}
#[test]
fn server_info_probe_round_does_not_count_no_client_slots_as_rpc_failures() {
let caches = vec![Mutex::new(PeerAdminCache::new())];
let endpoints = EndpointServerPools::default();
let probes = vec![PeerServerInfoProbe {
host: "node-a:9000".to_string(),
result: Err(PeerServerInfoProbeFailure::NoClient),
}];
let replies = publish_server_info_probe_round(&caches, &endpoints, probes);
assert_eq!(replies.len(), 1);
assert_eq!(replies[0].endpoint, "node-a:9000");
assert_eq!(replies[0].state, ItemState::Unknown.to_string());
assert_eq!(
caches[0]
.lock()
.expect("peer cache should lock after no-client publish")
.server_failures,
0
);
}
#[test]
fn endpoint_host_matches_direct_and_canonicalized() {
// Direct match (IP deployment): peer host already equals host_port.
@@ -141,6 +141,7 @@ impl MigrationBackendSpy {
stream: Box::new(Cursor::new(vec![0_u8; 3])),
object_info: ObjectInfo::default(),
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
}
}
+1
View File
@@ -8194,6 +8194,7 @@ mod tests {
..Default::default()
},
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
})
}
+56 -1
View File
@@ -206,7 +206,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
// });
// }
if object_info.size == 0 {
if object_info.size == 0 && !object_info.is_encrypted() {
record_get_object_reader_path_observation(GET_OBJECT_PATH_EMPTY, object_class, size_bucket);
// if let Some(rs) = range {
// let _ = rs.get_offset_length(object_info.size)?;
@@ -216,6 +216,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
stream: Box::new(Cursor::new(Vec::new())),
object_info,
buffered_body: Some(Bytes::new()),
resolved_sse: None,
body_source: GetObjectBodySource::Unprobed,
};
return Ok(reader);
@@ -303,6 +304,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
stream: Box::new(Cursor::new(body.clone())),
object_info,
buffered_body: Some(body),
resolved_sse: None,
body_source: GetObjectBodySource::Unprobed,
};
return Ok(reader);
@@ -381,6 +383,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
stream: Box::new(Cursor::new(body.clone())),
object_info,
buffered_body: Some(body),
resolved_sse: None,
body_source: GetObjectBodySource::Unprobed,
};
return Ok(reader);
@@ -462,6 +465,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
stream: Box::new(Cursor::new(body.clone())),
object_info,
buffered_body: Some(body),
resolved_sse: None,
body_source: GetObjectBodySource::HookServed,
};
if lock_optimization_enabled {
@@ -505,6 +509,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
stream: Box::new(Cursor::new(body.clone())),
object_info,
buffered_body: Some(body),
resolved_sse: None,
body_source,
};
if lock_optimization_enabled {
@@ -547,6 +552,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
stream: Box::new(Cursor::new(body.clone())),
object_info,
buffered_body: Some(body),
resolved_sse: None,
body_source,
};
if lock_optimization_enabled {
@@ -3311,6 +3317,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
stream: Box::new(TransitionUploadReader::new(pr, Arc::clone(&consumed))),
object_info: oi,
buffered_body: None,
resolved_sse: None,
body_source: GetObjectBodySource::Unprobed,
});
@@ -4047,6 +4054,54 @@ pub(in crate::set_disk::ops) mod hermetic_set_disks_support {
}
}
#[cfg(test)]
mod zero_length_encrypted_read_tests {
use super::hermetic_set_disks_support::hermetic_set_disks;
use super::*;
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
#[tokio::test]
async fn encrypted_zero_length_object_does_not_use_plain_empty_fast_path() {
let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks(4).await;
let bucket = "zero-length-encrypted-read";
let object = "empty";
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(Vec::new());
set_disks
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
.await
.expect("empty object should be written");
let metadata = HashMap::from([(rustfs_utils::http::SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]);
set_disks
.put_object_metadata(
bucket,
object,
&ObjectOptions {
eval_metadata: Some(metadata),
..Default::default()
},
)
.await
.expect("SSE-C metadata should be persisted");
let result = set_disks
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
.await;
let error = match result {
Ok(_) => panic!("encrypted empty object must validate SSE-C headers"),
Err(error) => error,
};
assert!(
error.to_string().contains("SSE-C") || error.to_string().contains("customer"),
"the encrypted read must fail at SSE-C validation, got: {error}"
);
}
}
#[cfg(test)]
mod metadata_mutation_generation_tests {
use super::hermetic_set_disks_support::hermetic_set_disks;
+950
View File
@@ -0,0 +1,950 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use aes_gcm::{
Aes256Gcm, Nonce,
aead::{Aead, KeyInit, Payload},
};
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use chacha20poly1305::ChaCha20Poly1305;
use hmac::{Hmac, Mac};
use rustfs_utils::http::{AMZ_SERVER_SIDE_ENCRYPTION, get_consistent_metadata_value};
use serde::Deserialize;
use sha2::Sha256;
use std::collections::HashMap;
#[cfg(not(any(test, debug_assertions)))]
use std::sync::OnceLock;
use thiserror::Error;
use zeroize::Zeroizing;
pub(crate) const DEFAULT_SSE_ALGORITHM: &str = "AES256";
const SSE_KMS_ALGORITHM: &str = "aws:kms";
pub(crate) const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id";
pub(crate) const INTERNAL_ENCRYPTION_KEY_HEADER: &str = "x-rustfs-encryption-key";
pub(crate) const INTERNAL_ENCRYPTION_IV_HEADER: &str = "x-rustfs-encryption-iv";
pub(crate) const AMZ_SERVER_SIDE_ENCRYPTION_KMS_KEY_ID: &str = "x-amz-server-side-encryption-aws-kms-key-id";
pub(crate) const MINIO_INTERNAL_ENCRYPTION_IV_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Iv";
pub(crate) const MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm";
pub(crate) const MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key";
pub(crate) const MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key";
pub(crate) const MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER: &str =
"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key";
pub(crate) const MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id";
pub(crate) const MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM: &str = "DAREv2-HMAC-SHA256";
const MINIO_STATIC_KMS_KEY_ENV: &str = "RUSTFS_MINIO_STATIC_KMS_KEY";
const MINIO_STATIC_KMS_RANDOM_SIZE: usize = 28;
const MINIO_STATIC_KMS_IV_SIZE: usize = 16;
const MINIO_STATIC_KMS_NONCE_SIZE: usize = 12;
type HmacSha256 = Hmac<Sha256>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ManagedSseScheme {
SseS3,
SseKms,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ManagedDekProvider {
LocalSseS3,
MinioKeyValue,
Kms,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PersistedManagedEncryption {
LegacySseS3Local,
LegacySseS3Kms,
LegacySseKms,
MinioSseS3KeyValue,
MinioSseS3Kms,
MinioSseKmsKms,
}
impl PersistedManagedEncryption {
pub fn scheme(self) -> ManagedSseScheme {
match self {
Self::LegacySseS3Local | Self::LegacySseS3Kms | Self::MinioSseS3KeyValue | Self::MinioSseS3Kms => {
ManagedSseScheme::SseS3
}
Self::LegacySseKms | Self::MinioSseKmsKms => ManagedSseScheme::SseKms,
}
}
pub fn provider(self) -> ManagedDekProvider {
match self {
Self::LegacySseS3Local => ManagedDekProvider::LocalSseS3,
Self::MinioSseS3KeyValue => ManagedDekProvider::MinioKeyValue,
Self::LegacySseS3Kms | Self::LegacySseKms | Self::MinioSseS3Kms | Self::MinioSseKmsKms => ManagedDekProvider::Kms,
}
}
pub fn uses_object_key(self) -> bool {
matches!(self, Self::MinioSseS3KeyValue | Self::MinioSseS3Kms | Self::MinioSseKmsKms)
}
}
#[derive(Debug, Clone, Error, PartialEq, Eq)]
pub enum PersistedEncryptionError {
#[error("conflicting values for encryption metadata {field}")]
ConflictingValue { field: &'static str },
#[error("conflicting MinIO managed SSE format markers")]
ConflictingMinioMarkers,
#[error("incomplete managed SSE metadata: missing {field}")]
MissingField { field: &'static str },
#[error("unsupported stored server-side encryption {algorithm}")]
UnsupportedSseAlgorithm { algorithm: String },
#[error("unsupported MinIO seal algorithm {algorithm}")]
UnsupportedSealAlgorithm { algorithm: String },
#[error("stored SSE algorithm conflicts with the persisted encryption format")]
SchemeConflict,
#[error("conflicting encrypted data-key metadata")]
ConflictingEncryptedDataKey,
#[error("conflicting KMS key-id metadata")]
ConflictingKmsKeyId,
#[error("invalid MinIO static KMS configuration: {reason}")]
InvalidMinioStaticKmsConfiguration { reason: String },
#[error("invalid MinIO static KMS ciphertext: {reason}")]
InvalidMinioStaticKmsCiphertext { reason: String },
#[error("unrecognized legacy managed SSE encrypted data-key format")]
UnknownLegacyEnvelope,
}
pub fn classify_persisted_managed_encryption(
metadata: &HashMap<String, String>,
encrypted_dek: &[u8],
) -> Result<PersistedManagedEncryption, PersistedEncryptionError> {
let public_algorithm = consistent(metadata, AMZ_SERVER_SIDE_ENCRYPTION)?;
if let Some(algorithm) = public_algorithm
&& !matches!(algorithm, DEFAULT_SSE_ALGORITHM | SSE_KMS_ALGORITHM)
{
return Err(PersistedEncryptionError::UnsupportedSseAlgorithm {
algorithm: algorithm.to_string(),
});
}
let s3_sealed_key = consistent(metadata, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)?;
let kms_sealed_key = consistent(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)?;
if s3_sealed_key.is_some() && kms_sealed_key.is_some() {
return Err(PersistedEncryptionError::ConflictingMinioMarkers);
}
if s3_sealed_key.is_some() || kms_sealed_key.is_some() {
if s3_sealed_key.is_some() {
require_non_empty(s3_sealed_key, MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)?;
}
if kms_sealed_key.is_some() {
require_non_empty(kms_sealed_key, MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER)?;
}
require_non_empty(
consistent(metadata, MINIO_INTERNAL_ENCRYPTION_IV_HEADER)?,
MINIO_INTERNAL_ENCRYPTION_IV_HEADER,
)?;
let seal_algorithm = require_non_empty(
consistent(metadata, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER)?,
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
)?;
if seal_algorithm != MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM {
return Err(PersistedEncryptionError::UnsupportedSealAlgorithm {
algorithm: seal_algorithm.to_string(),
});
}
let kms_key_id = consistent(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER)?;
let encrypted_data_key = consistent(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER)?;
let has_kms_pair = match (kms_key_id, encrypted_data_key) {
(Some(key_id), Some(data_key)) => {
require_non_empty(Some(key_id), MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER)?;
require_non_empty(Some(data_key), MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER)?;
true
}
(None, None) => false,
(None, Some(_)) => {
return Err(PersistedEncryptionError::MissingField {
field: MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
});
}
(Some(_), None) => {
return Err(PersistedEncryptionError::MissingField {
field: MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
});
}
};
validate_encrypted_key_alias(metadata)?;
validate_kms_key_id_alias(metadata)?;
return match (s3_sealed_key, kms_sealed_key, public_algorithm) {
(Some(_), None, Some(SSE_KMS_ALGORITHM)) | (None, Some(_), Some(DEFAULT_SSE_ALGORITHM)) => {
Err(PersistedEncryptionError::SchemeConflict)
}
(Some(_), None, _) if !has_kms_pair => Ok(PersistedManagedEncryption::MinioSseS3KeyValue),
(Some(_), None, _) => Ok(PersistedManagedEncryption::MinioSseS3Kms),
(None, Some(_), _) if !has_kms_pair => Err(PersistedEncryptionError::MissingField {
field: MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER,
}),
(None, Some(_), _) => Ok(PersistedManagedEncryption::MinioSseKmsKms),
_ => Err(PersistedEncryptionError::ConflictingMinioMarkers),
};
}
if consistent(metadata, MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER)?.is_some()
|| consistent(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER)?.is_some()
{
return Err(PersistedEncryptionError::MissingField {
field: MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
});
}
require_non_empty(consistent(metadata, INTERNAL_ENCRYPTION_KEY_HEADER)?, INTERNAL_ENCRYPTION_KEY_HEADER)?;
require_non_empty(consistent(metadata, INTERNAL_ENCRYPTION_IV_HEADER)?, INTERNAL_ENCRYPTION_IV_HEADER)?;
match public_algorithm {
Some(SSE_KMS_ALGORITHM) if is_local_sse_s3_envelope(encrypted_dek) => Err(PersistedEncryptionError::SchemeConflict),
Some(SSE_KMS_ALGORITHM) => Ok(PersistedManagedEncryption::LegacySseKms),
Some(DEFAULT_SSE_ALGORITHM) | None if rustfs_kms::is_data_key_envelope(encrypted_dek) => {
// RUSTFS_COMPAT_TODO(rustfs-5063): older SSE-S3 objects may contain KMS-wrapped DEKs. Remove after every referenced legacy DEK has been rewrapped with the local SSE-S3 provider.
Ok(PersistedManagedEncryption::LegacySseS3Kms)
}
Some(DEFAULT_SSE_ALGORITHM) | None if is_local_sse_s3_envelope(encrypted_dek) => {
Ok(PersistedManagedEncryption::LegacySseS3Local)
}
Some(DEFAULT_SSE_ALGORITHM) | None => Err(PersistedEncryptionError::UnknownLegacyEnvelope),
Some(_) => unreachable!("unsupported algorithms return before format classification"),
}
}
fn consistent<'a>(
metadata: &'a HashMap<String, String>,
field: &'static str,
) -> Result<Option<&'a str>, PersistedEncryptionError> {
get_consistent_metadata_value(metadata, field).map_err(|_| PersistedEncryptionError::ConflictingValue { field })
}
fn require_non_empty<'a>(value: Option<&'a str>, field: &'static str) -> Result<&'a str, PersistedEncryptionError> {
value
.filter(|value| !value.is_empty())
.ok_or(PersistedEncryptionError::MissingField { field })
}
fn validate_encrypted_key_alias(metadata: &HashMap<String, String>) -> Result<(), PersistedEncryptionError> {
let Some(rustfs_key) = consistent(metadata, INTERNAL_ENCRYPTION_KEY_HEADER)? else {
return Ok(());
};
let minio_key = require_non_empty(
consistent(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER)?,
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
)?;
if rustfs_key != minio_key {
return Err(PersistedEncryptionError::ConflictingEncryptedDataKey);
}
Ok(())
}
fn validate_kms_key_id_alias(metadata: &HashMap<String, String>) -> Result<(), PersistedEncryptionError> {
let values = [
consistent(metadata, INTERNAL_ENCRYPTION_KEY_ID_HEADER)?,
consistent(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER)?,
consistent(metadata, AMZ_SERVER_SIDE_ENCRYPTION_KMS_KEY_ID)?,
];
let mut present = values.into_iter().flatten().filter(|value| !value.is_empty());
let Some(first) = present.next() else {
return Ok(());
};
if present.any(|value| value != first) {
return Err(PersistedEncryptionError::ConflictingKmsKeyId);
}
Ok(())
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct MinioStaticKmsJsonCiphertext {
#[serde(rename = "aead")]
algorithm: String,
id: Option<String>,
iv: String,
nonce: String,
bytes: String,
}
#[derive(Clone, Copy)]
enum MinioStaticKmsAlgorithm {
Aes256,
ChaCha20,
}
#[derive(Clone)]
struct ConfiguredMinioStaticKmsKey {
id: String,
material: Zeroizing<[u8; 32]>,
}
struct ParsedMinioStaticKmsCiphertext {
ciphertext: Vec<u8>,
iv: [u8; 16],
nonce: [u8; 12],
algorithm: MinioStaticKmsAlgorithm,
}
pub fn decrypt_minio_static_kms_dek(
kms_key_id: &str,
encrypted_dek: &[u8],
context: &HashMap<String, String>,
) -> Result<Option<[u8; 32]>, PersistedEncryptionError> {
let Some(configured_key) = minio_static_kms_key()? else {
return Ok(None);
};
if configured_key.id != kms_key_id {
return Ok(None);
}
let parsed = parse_minio_static_kms_ciphertext(encrypted_dek)?;
let ParsedMinioStaticKmsCiphertext {
ciphertext,
iv,
nonce,
algorithm,
} = parsed;
let master_key = configured_key.material;
let sealing_key: Zeroizing<[u8; 32]> = Zeroizing::new(match algorithm {
MinioStaticKmsAlgorithm::Aes256 => {
let mut mac = HmacSha256::new_from_slice(master_key.as_slice()).map_err(|err| {
PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
reason: format!("invalid HMAC key: {err}"),
}
})?;
mac.update(&iv);
mac.finalize().into_bytes().into()
}
MinioStaticKmsAlgorithm::ChaCha20 => chacha20::hchacha::<chacha20::R20>((&*master_key).into(), (&iv).into()).into(),
});
let associated_data = marshal_minio_kms_context(context);
let plaintext = Zeroizing::new(
match algorithm {
MinioStaticKmsAlgorithm::Aes256 => Aes256Gcm::new_from_slice(sealing_key.as_slice())
.map_err(|err| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
reason: format!("invalid AES sealing key: {err}"),
})?
.decrypt(
&Nonce::from(nonce),
Payload {
msg: &ciphertext,
aad: &associated_data,
},
),
MinioStaticKmsAlgorithm::ChaCha20 => ChaCha20Poly1305::new_from_slice(sealing_key.as_slice())
.map_err(|err| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
reason: format!("invalid ChaCha20 sealing key: {err}"),
})?
.decrypt(
&chacha20poly1305::Nonce::from(nonce),
Payload {
msg: &ciphertext,
aad: &associated_data,
},
),
}
.map_err(|_| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
reason: "AEAD authentication failed".to_string(),
})?,
);
plaintext
.as_slice()
.try_into()
.map(Some)
.map_err(|_| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
reason: "plaintext data key must be 32 bytes".to_string(),
})
}
fn minio_static_kms_key() -> Result<Option<ConfiguredMinioStaticKmsKey>, PersistedEncryptionError> {
#[cfg(not(any(test, debug_assertions)))]
{
static CONFIG: OnceLock<Result<Option<ConfiguredMinioStaticKmsKey>, PersistedEncryptionError>> = OnceLock::new();
return CONFIG.get_or_init(parse_minio_static_kms_key).clone();
}
#[cfg(any(test, debug_assertions))]
parse_minio_static_kms_key()
}
fn parse_minio_static_kms_key() -> Result<Option<ConfiguredMinioStaticKmsKey>, PersistedEncryptionError> {
let Some(value) = std::env::var_os(MINIO_STATIC_KMS_KEY_ENV) else {
return Ok(None);
};
let value =
Zeroizing::new(
value
.into_string()
.map_err(|_| PersistedEncryptionError::InvalidMinioStaticKmsConfiguration {
reason: format!("{MINIO_STATIC_KMS_KEY_ENV} must be valid UTF-8"),
})?,
);
let (key_id, encoded_key) =
value
.split_once(':')
.ok_or_else(|| PersistedEncryptionError::InvalidMinioStaticKmsConfiguration {
reason: format!("{MINIO_STATIC_KMS_KEY_ENV} must use <key-id>:<base64-key>"),
})?;
if key_id.is_empty() {
return Err(PersistedEncryptionError::InvalidMinioStaticKmsConfiguration {
reason: "key ID must not be empty".to_string(),
});
}
let decoded_key = Zeroizing::new(BASE64_STANDARD.decode(encoded_key).map_err(|_| {
PersistedEncryptionError::InvalidMinioStaticKmsConfiguration {
reason: "key material must be valid Base64".to_string(),
}
})?);
let key: [u8; 32] =
decoded_key
.as_slice()
.try_into()
.map_err(|_| PersistedEncryptionError::InvalidMinioStaticKmsConfiguration {
reason: "key material must decode to exactly 32 bytes".to_string(),
})?;
if key == [0u8; 32] {
return Err(PersistedEncryptionError::InvalidMinioStaticKmsConfiguration {
reason: "key material must not be all zero".to_string(),
});
}
Ok(Some(ConfiguredMinioStaticKmsKey {
id: key_id.to_string(),
material: Zeroizing::new(key),
}))
}
fn parse_minio_static_kms_ciphertext(encrypted_dek: &[u8]) -> Result<ParsedMinioStaticKmsCiphertext, PersistedEncryptionError> {
if encrypted_dek.first() == Some(&b'{') && encrypted_dek.last() == Some(&b'}') {
let envelope: MinioStaticKmsJsonCiphertext =
serde_json::from_slice(encrypted_dek).map_err(|err| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
reason: format!("invalid legacy JSON: {err}"),
})?;
let _ = envelope.id;
let algorithm = match envelope.algorithm.as_str() {
"AES-256-GCM-HMAC-SHA-256" => MinioStaticKmsAlgorithm::Aes256,
"ChaCha20Poly1305" => MinioStaticKmsAlgorithm::ChaCha20,
algorithm => {
return Err(PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
reason: format!("unsupported algorithm {algorithm}"),
});
}
};
return Ok(ParsedMinioStaticKmsCiphertext {
ciphertext: decode_minio_static_kms_field("bytes", &envelope.bytes)?,
iv: decode_minio_static_kms_array("iv", &envelope.iv)?,
nonce: decode_minio_static_kms_array("nonce", &envelope.nonce)?,
algorithm,
});
}
if encrypted_dek.len() <= MINIO_STATIC_KMS_RANDOM_SIZE {
return Err(PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
reason: "binary ciphertext is too short".to_string(),
});
}
let split_at = encrypted_dek.len() - MINIO_STATIC_KMS_RANDOM_SIZE;
let (ciphertext, random) = encrypted_dek.split_at(split_at);
Ok(ParsedMinioStaticKmsCiphertext {
ciphertext: ciphertext.to_vec(),
iv: random[..MINIO_STATIC_KMS_IV_SIZE].try_into().map_err(|_| {
PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
reason: "invalid binary IV length".to_string(),
}
})?,
nonce: random[MINIO_STATIC_KMS_IV_SIZE..MINIO_STATIC_KMS_IV_SIZE + MINIO_STATIC_KMS_NONCE_SIZE]
.try_into()
.map_err(|_| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
reason: "invalid binary nonce length".to_string(),
})?,
algorithm: MinioStaticKmsAlgorithm::Aes256,
})
}
fn decode_minio_static_kms_field(field: &'static str, value: &str) -> Result<Vec<u8>, PersistedEncryptionError> {
BASE64_STANDARD
.decode(value)
.map_err(|_| PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
reason: format!("{field} must be valid Base64"),
})
}
fn decode_minio_static_kms_array<const N: usize>(field: &'static str, value: &str) -> Result<[u8; N], PersistedEncryptionError> {
decode_minio_static_kms_field(field, value)?.try_into().map_err(|_| {
PersistedEncryptionError::InvalidMinioStaticKmsCiphertext {
reason: format!("{field} must decode to exactly {N} bytes"),
}
})
}
fn marshal_minio_kms_context(context: &HashMap<String, String>) -> Vec<u8> {
let mut entries: Vec<_> = context.iter().collect();
entries.sort_by_key(|(key, _)| *key);
let mut json = String::from("{");
for (index, (key, value)) in entries.into_iter().enumerate() {
if index > 0 {
json.push(',');
}
push_minio_json_string(&mut json, key);
json.push(':');
push_minio_json_string(&mut json, value);
}
json.push('}');
json.into_bytes()
}
fn push_minio_json_string(output: &mut String, value: &str) {
output.push('"');
for character in value.chars() {
match character {
'"' => output.push_str("\\\""),
'\\' => output.push_str("\\\\"),
'\n' => output.push_str("\\n"),
'\r' => output.push_str("\\r"),
'\t' => output.push_str("\\t"),
'<' => output.push_str("\\u003c"),
'>' => output.push_str("\\u003e"),
'&' => output.push_str("\\u0026"),
'\u{2028}' => output.push_str("\\u2028"),
'\u{2029}' => output.push_str("\\u2029"),
character if character <= '\u{1f}' => {
const HEX: &[u8; 16] = b"0123456789abcdef";
let byte = character as u8;
output.push_str("\\u00");
output.push(HEX[(byte >> 4) as usize] as char);
output.push(HEX[(byte & 0x0f) as usize] as char);
}
character => output.push(character),
}
}
output.push('"');
}
fn is_local_sse_s3_envelope(encrypted_dek: &[u8]) -> bool {
let Ok(encoded) = std::str::from_utf8(encrypted_dek) else {
return false;
};
let Some((nonce, ciphertext)) = encoded.split_once(':') else {
return false;
};
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
BASE64_STANDARD.decode(nonce).is_ok_and(|nonce| nonce.len() == 12)
&& BASE64_STANDARD
.decode(ciphertext)
.is_ok_and(|ciphertext| ciphertext.len() == 48)
}
#[cfg(test)]
mod tests {
use super::*;
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
fn local_envelope() -> Vec<u8> {
format!("{}:{}", BASE64_STANDARD.encode([1u8; 12]), BASE64_STANDARD.encode([2u8; 48])).into_bytes()
}
fn kms_envelope() -> Vec<u8> {
serde_json::to_vec(&serde_json::json!({
"key_id": "data-key",
"master_key_id": "master-key",
"key_spec": "AES_256",
"encrypted_key": [1, 2, 3, 4],
"nonce": [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
"encryption_context": {},
"created_at": "2024-01-01T00:00:00+00:00"
}))
.expect("serialize KMS envelope fixture")
}
fn legacy_metadata(algorithm: Option<&str>, encrypted_dek: &[u8]) -> HashMap<String, String> {
let mut metadata = HashMap::from([
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode(encrypted_dek)),
(INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([3u8; 12])),
]);
if let Some(algorithm) = algorithm {
metadata.insert(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), algorithm.to_string());
}
metadata
}
fn minio_metadata(kms: bool, encrypted_dek: &[u8]) -> HashMap<String, String> {
let object_key_header = if kms {
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER
} else {
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER
};
let mut metadata = HashMap::from([
(object_key_header.to_string(), BASE64_STANDARD.encode([4u8; 64])),
(
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(),
BASE64_STANDARD.encode(encrypted_dek),
),
(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), "default".to_string()),
(MINIO_INTERNAL_ENCRYPTION_IV_HEADER.to_string(), BASE64_STANDARD.encode([5u8; 32])),
(
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
MINIO_INTERNAL_ENCRYPTION_SEAL_ALGORITHM.to_string(),
),
]);
metadata.insert(
AMZ_SERVER_SIDE_ENCRYPTION.to_string(),
if kms { SSE_KMS_ALGORITHM } else { DEFAULT_SSE_ALGORITHM }.to_string(),
);
metadata
}
#[test]
fn classifies_released_legacy_formats() {
let local = local_envelope();
let kms = kms_envelope();
assert_eq!(
classify_persisted_managed_encryption(&legacy_metadata(Some(DEFAULT_SSE_ALGORITHM), &local), &local)
.expect("classify local Direct format"),
PersistedManagedEncryption::LegacySseS3Local
);
assert_eq!(
classify_persisted_managed_encryption(&legacy_metadata(Some(DEFAULT_SSE_ALGORITHM), &kms), &kms)
.expect("classify legacy SSE-S3 KMS envelope"),
PersistedManagedEncryption::LegacySseS3Kms
);
assert_eq!(
classify_persisted_managed_encryption(
&legacy_metadata(Some(SSE_KMS_ALGORITHM), b"opaque-kms-key"),
b"opaque-kms-key"
)
.expect("classify opaque SSE-KMS envelope"),
PersistedManagedEncryption::LegacySseKms
);
}
#[test]
fn minio_markers_override_envelope_shape() {
let local = local_envelope();
let kms = kms_envelope();
assert_eq!(
classify_persisted_managed_encryption(&minio_metadata(false, &kms), &kms)
.expect("SSE-S3 marker selects MinIO SSE-S3"),
PersistedManagedEncryption::MinioSseS3Kms
);
assert_eq!(
classify_persisted_managed_encryption(&minio_metadata(true, &local), &local)
.expect("SSE-KMS marker selects MinIO SSE-KMS"),
PersistedManagedEncryption::MinioSseKmsKms
);
}
#[test]
fn accepts_minio_sse_s3_key_value_only_when_kms_pair_is_absent() {
let local = local_envelope();
let mut metadata = minio_metadata(false, &local);
metadata.remove(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER);
metadata.remove(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER);
assert_eq!(
classify_persisted_managed_encryption(&metadata, &[]).expect("classify MinIO SSE-S3 K/V metadata"),
PersistedManagedEncryption::MinioSseS3KeyValue
);
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER.to_string(), String::new());
metadata.insert(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER.to_string(), String::new());
assert_eq!(
classify_persisted_managed_encryption(&metadata, &[]),
Err(PersistedEncryptionError::MissingField {
field: MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER
})
);
}
#[test]
fn rejects_minio_sse_kms_without_kms_pair() {
let local = local_envelope();
let mut metadata = minio_metadata(true, &local);
metadata.remove(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER);
metadata.remove(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER);
assert_eq!(
classify_persisted_managed_encryption(&metadata, &[]),
Err(PersistedEncryptionError::MissingField {
field: MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER
})
);
}
#[test]
fn persisted_minio_provider_does_not_depend_on_runtime_kms_availability() {
let static_kms = br#"{"aead":"AES-256-GCM-HMAC-SHA-256","iv":"AQEBAQEBAQEBAQEBAQEBAQ==","nonce":"AgICAgICAgICAgIC","bytes":"AwMDAw=="}"#;
let opaque_kms = b"opaque-external-kms-ciphertext";
assert_eq!(
classify_persisted_managed_encryption(&minio_metadata(false, static_kms), static_kms)
.expect("classify MinIO static KMS envelope")
.provider(),
ManagedDekProvider::Kms
);
assert_eq!(
classify_persisted_managed_encryption(&minio_metadata(true, opaque_kms), opaque_kms)
.expect("classify external KMS ciphertext")
.provider(),
ManagedDekProvider::Kms
);
let mut key_value = minio_metadata(false, opaque_kms);
key_value.remove(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER);
key_value.remove(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER);
assert_eq!(
classify_persisted_managed_encryption(&key_value, &[])
.expect("classify historical K/V provider")
.provider(),
ManagedDekProvider::MinioKeyValue
);
}
#[test]
fn decrypts_minio_static_kms_binary_aes_ciphertext() {
let master_key = [0x31; 32];
let plaintext_key = [0x52; 32];
let iv = [0x63; 16];
let nonce = [0x74; 12];
let context = HashMap::from([("bucket".to_string(), "bucket/object".to_string())]);
let mut mac = HmacSha256::new_from_slice(&master_key).expect("valid master key");
mac.update(&iv);
let sealing_key = mac.finalize().into_bytes();
let ciphertext = Aes256Gcm::new_from_slice(sealing_key.as_slice())
.expect("valid sealing key")
.encrypt(
&Nonce::from(nonce),
Payload {
msg: &plaintext_key,
aad: &marshal_minio_kms_context(&context),
},
)
.expect("encrypt fixture");
let mut envelope = ciphertext;
envelope.extend_from_slice(&iv);
envelope.extend_from_slice(&nonce);
let configured_key = format!("minio-key:{}", BASE64_STANDARD.encode(master_key));
temp_env::with_var(MINIO_STATIC_KMS_KEY_ENV, Some(configured_key), || {
assert_eq!(
decrypt_minio_static_kms_dek("minio-key", &envelope, &context).expect("decrypt binary ciphertext"),
Some(plaintext_key)
);
assert_eq!(
decrypt_minio_static_kms_dek("external-key", &envelope, &context)
.expect("different key ID must remain external KMS"),
None
);
});
}
#[test]
fn decrypts_official_minio_legacy_chacha20_ciphertext() {
let configured_key = "my-key:eEm+JI9/q4JhH8QwKvf3LKo4DEBl6QbfvAl1CAbMIv8=";
let ciphertext = br#"{"aead":"ChaCha20Poly1305","iv":"JbI+vwvYww1lCb5VpkAFuQ==","nonce":"ARjIjJxBSD541Gz8","bytes":"KCbEc2sA0TLvA7aWTWa23AdccVfJMpOxwgG8hm+4PaNrxYfy1xFWZg2gEenVrOgv"}"#;
let expected: [u8; 32] = BASE64_STANDARD
.decode("zmS7NrG765UZ0ZN85oPjybelxqVvpz01vxsSpOISy2M=")
.expect("decode official plaintext")
.try_into()
.expect("official plaintext is 32 bytes");
temp_env::with_var(MINIO_STATIC_KMS_KEY_ENV, Some(configured_key), || {
assert_eq!(
decrypt_minio_static_kms_dek("my-key", ciphertext, &HashMap::new()).expect("decrypt official MinIO ciphertext"),
Some(expected)
);
});
}
#[test]
fn rejects_malformed_config_and_unknown_legacy_fields() {
temp_env::with_var(MINIO_STATIC_KMS_KEY_ENV, Some("missing-separator"), || {
assert!(matches!(
decrypt_minio_static_kms_dek("my-key", b"ciphertext", &HashMap::new()),
Err(PersistedEncryptionError::InvalidMinioStaticKmsConfiguration { .. })
));
});
let configured_key = format!("my-key:{}", BASE64_STANDARD.encode([0x31; 32]));
let ciphertext =
br#"{"aead":"AES-256-GCM-HMAC-SHA-256","iv":"Y2NjY2NjY2NjY2NjY2NjYw==","nonce":"dHR0dHR0dHR0dHR0","bytes":"AA==","extra":true}"#;
temp_env::with_var(MINIO_STATIC_KMS_KEY_ENV, Some(configured_key), || {
assert!(matches!(
decrypt_minio_static_kms_dek("my-key", ciphertext, &HashMap::new()),
Err(PersistedEncryptionError::InvalidMinioStaticKmsCiphertext { .. })
));
});
}
#[test]
fn rejects_all_zero_minio_static_kms_key() {
let configured_key = format!("my-key:{}", BASE64_STANDARD.encode([0u8; 32]));
temp_env::with_var(MINIO_STATIC_KMS_KEY_ENV, Some(configured_key), || {
assert!(matches!(
decrypt_minio_static_kms_dek("my-key", b"ciphertext", &HashMap::new()),
Err(PersistedEncryptionError::InvalidMinioStaticKmsConfiguration { .. })
));
});
}
#[test]
fn marshals_minio_kms_context_with_go_json_escaping() {
let context = HashMap::from([
("z".to_string(), "line\n".to_string()),
("<\u{8}".to_string(), "&\u{2028}".to_string()),
]);
assert_eq!(marshal_minio_kms_context(&context), br#"{"\u003c\u0008":"\u0026\u2028","z":"line\n"}"#);
}
#[test]
fn rejects_conflicting_or_partial_minio_metadata() {
let local = local_envelope();
let mut both = minio_metadata(false, &local);
both.insert(
MINIO_INTERNAL_ENCRYPTION_KMS_SEALED_KEY_HEADER.to_string(),
BASE64_STANDARD.encode([6u8; 64]),
);
assert_eq!(
classify_persisted_managed_encryption(&both, &local),
Err(PersistedEncryptionError::ConflictingMinioMarkers)
);
let mut wrong_scheme = minio_metadata(true, &local);
wrong_scheme.insert(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), DEFAULT_SSE_ALGORITHM.to_string());
assert_eq!(
classify_persisted_managed_encryption(&wrong_scheme, &local),
Err(PersistedEncryptionError::SchemeConflict)
);
let mut partial = minio_metadata(false, &local);
partial.remove(MINIO_INTERNAL_ENCRYPTION_IV_HEADER);
assert_eq!(
classify_persisted_managed_encryption(&partial, &local),
Err(PersistedEncryptionError::MissingField {
field: MINIO_INTERNAL_ENCRYPTION_IV_HEADER
})
);
let mut missing_key_id = minio_metadata(false, &local);
missing_key_id.remove(MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER);
assert_eq!(
classify_persisted_managed_encryption(&missing_key_id, &local),
Err(PersistedEncryptionError::MissingField {
field: MINIO_INTERNAL_ENCRYPTION_KMS_KEY_ID_HEADER
})
);
let mut missing_data_key = minio_metadata(false, &local);
missing_data_key.remove(MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER);
assert_eq!(
classify_persisted_managed_encryption(&missing_data_key, &local),
Err(PersistedEncryptionError::MissingField {
field: MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER
})
);
let mut unknown_algorithm = minio_metadata(false, &local);
unknown_algorithm.insert(
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER.to_string(),
"future-seal-algorithm".to_string(),
);
assert_eq!(
classify_persisted_managed_encryption(&unknown_algorithm, &local),
Err(PersistedEncryptionError::UnsupportedSealAlgorithm {
algorithm: "future-seal-algorithm".to_string()
})
);
for field in [
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER,
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
] {
let mut empty = minio_metadata(false, &local);
empty.insert(field.to_string(), String::new());
assert_eq!(
classify_persisted_managed_encryption(&empty, &local),
Err(PersistedEncryptionError::MissingField { field })
);
}
for field in [
MINIO_INTERNAL_ENCRYPTION_ALGORITHM_HEADER,
MINIO_INTERNAL_ENCRYPTION_KMS_DATA_KEY_HEADER,
] {
let metadata = HashMap::from([(field.to_string(), "orphaned".to_string())]);
assert_eq!(
classify_persisted_managed_encryption(&metadata, &local),
Err(PersistedEncryptionError::MissingField {
field: MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER
})
);
}
}
#[test]
fn rejects_unknown_legacy_envelope_and_conflicting_alias() {
let unknown = b"not-an-envelope";
assert_eq!(
classify_persisted_managed_encryption(&legacy_metadata(Some(DEFAULT_SSE_ALGORITHM), unknown), unknown),
Err(PersistedEncryptionError::UnknownLegacyEnvelope)
);
let local = local_envelope();
let mut conflicting = minio_metadata(false, &local);
conflicting.insert(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), BASE64_STANDARD.encode(b"different"));
assert_eq!(
classify_persisted_managed_encryption(&conflicting, &local),
Err(PersistedEncryptionError::ConflictingEncryptedDataKey)
);
let mut conflicting_key_id = minio_metadata(false, &local);
conflicting_key_id.insert(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "different-key".to_string());
assert_eq!(
classify_persisted_managed_encryption(&conflicting_key_id, &local),
Err(PersistedEncryptionError::ConflictingKmsKeyId)
);
assert_eq!(
classify_persisted_managed_encryption(&legacy_metadata(Some(SSE_KMS_ALGORITHM), &local), &local),
Err(PersistedEncryptionError::SchemeConflict)
);
}
#[test]
fn accepts_case_insensitive_minio_markers_and_rejects_conflicting_duplicates() {
let local = local_envelope();
let mut metadata = minio_metadata(false, &local);
let sealed_key = metadata
.remove(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER)
.expect("MinIO fixture contains the sealed-key marker");
metadata.insert(MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_ascii_lowercase(), sealed_key);
assert_eq!(
classify_persisted_managed_encryption(&metadata, &local).expect("classify mixed-case MinIO metadata"),
PersistedManagedEncryption::MinioSseS3Kms
);
metadata.insert(
MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER.to_string(),
BASE64_STANDARD.encode([0x77u8; 64]),
);
assert_eq!(
classify_persisted_managed_encryption(&metadata, &local),
Err(PersistedEncryptionError::ConflictingValue {
field: MINIO_INTERNAL_ENCRYPTION_S3_SEALED_KEY_HEADER
})
);
}
}
+1
View File
@@ -642,6 +642,7 @@ mod tests {
stream: Box::new(Cursor::new(self.read_payload.clone())),
object_info: self.object_info(bucket, object, self.read_payload.len()),
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
})
}
+4
View File
@@ -2646,6 +2646,7 @@ mod tests {
stream: Box::new(Cursor::new(Vec::<u8>::new())),
object_info: ObjectInfo::default(),
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
};
@@ -2686,6 +2687,7 @@ mod tests {
stream: Box::new(Cursor::new(vec![1, 2, 3])),
object_info: ObjectInfo::default(),
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
};
@@ -2723,6 +2725,7 @@ mod tests {
stream: Box::new(Cursor::new(vec![1, 2, 3])),
object_info: ObjectInfo::default(),
buffered_body: Some(Bytes::from_static(b"123")),
resolved_sse: None,
body_source: Default::default(),
};
@@ -2760,6 +2763,7 @@ mod tests {
stream: Box::new(Cursor::new(vec![1, 2, 3])),
object_info: ObjectInfo::default(),
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
};
+170 -53
View File
@@ -7,10 +7,12 @@ use std::path::{Path, PathBuf};
mod storage_api;
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
use rustfs_utils::HashAlgorithm;
use serde::Deserialize;
use sha2::{Digest, Sha256};
use storage_api::minio_generated_read::{
DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
DiskAPI as _, DiskOption, Endpoint, Erasure, GetObjectReader, HTTPRangeSpec, ObjectInfo, ObjectOptions, create_bitrot_reader,
new_disk,
};
use temp_env::async_with_vars;
use tokio::io::{AsyncReadExt, AsyncWrite};
@@ -22,6 +24,11 @@ struct ManifestRecord {
backend_files: Vec<String>,
}
#[derive(Debug, Deserialize)]
struct RequestRecord {
headers: std::collections::HashMap<String, String>,
}
#[derive(Default)]
struct VecAsyncWriter {
bytes: Vec<u8>,
@@ -56,21 +63,17 @@ fn case_dir(case_id: &str) -> PathBuf {
fixture_root().join("cases").join(case_id)
}
fn beta5_fixture_root() -> PathBuf {
std::env::var_os("RUSTFS_BETA5_FIXTURE_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/rustfs-beta5-generated"))
}
fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> T {
let text = fs::read_to_string(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()));
serde_json::from_str(&text).unwrap_or_else(|err| panic!("parse {}: {err}", path.display()))
}
fn require_fixture_case(case_id: &str) -> PathBuf {
let path = case_dir(case_id);
assert!(
path.is_dir(),
"fixture case missing: {}. Run scripts/minio_fixture_lab/lab.py capture-matrix first.",
path.display()
);
path
}
fn read_plaintext_sha256(case_dir: &Path) -> String {
fs::read_to_string(case_dir.join("plaintext.sha256"))
.unwrap_or_else(|err| panic!("read plaintext.sha256 under {}: {err}", case_dir.display()))
@@ -78,9 +81,9 @@ fn read_plaintext_sha256(case_dir: &Path) -> String {
.to_string()
}
fn minio_static_kms_key_b64() -> String {
std::env::var("RUSTFS_MINIO_STATIC_KMS_KEY_B64")
.unwrap_or_else(|_| panic!("RUSTFS_MINIO_STATIC_KMS_KEY_B64 must point to the 32-byte static MinIO KMS key"))
fn minio_static_kms_key() -> String {
std::env::var("RUSTFS_MINIO_STATIC_KMS_KEY")
.unwrap_or_else(|_| panic!("RUSTFS_MINIO_STATIC_KMS_KEY must use <key-id>:<base64-32-byte-key>"))
}
fn object_xl_meta_path(case_dir: &Path, manifest: &ManifestRecord) -> PathBuf {
@@ -117,37 +120,53 @@ fn sha256_hex(bytes: &[u8]) -> String {
hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower)
}
async fn load_fixture_reader_input(case_id: &str) -> (ObjectInfo, Vec<u8>, String) {
let case_dir = require_fixture_case(case_id);
async fn load_fixture_reader_input(case_id: &str) -> (ObjectInfo, Vec<u8>, String, http::HeaderMap) {
load_fixture_reader_input_from(case_dir(case_id)).await
}
async fn load_fixture_reader_input_from(case_dir: PathBuf) -> (ObjectInfo, Vec<u8>, String, http::HeaderMap) {
assert!(case_dir.is_dir(), "fixture case missing: {}", case_dir.display());
let manifest: ManifestRecord = read_json(&case_dir.join("manifest.json"));
let request: RequestRecord = read_json(&case_dir.join("request.json"));
let expected_sha256 = read_plaintext_sha256(&case_dir);
let file_info = load_file_info(&case_dir, &manifest);
let encrypted = encrypted_fixture_bytes(&case_dir, &manifest, &file_info).await;
let object_info = load_object_info(&file_info, &manifest);
(object_info, encrypted, expected_sha256)
let mut headers = http::HeaderMap::new();
for (name, value) in request.headers {
let name = http::header::HeaderName::from_bytes(name.as_bytes())
.unwrap_or_else(|err| panic!("invalid fixture request header {name}: {err}"));
let value =
http::HeaderValue::try_from(value).unwrap_or_else(|err| panic!("invalid fixture request header value: {err}"));
headers.insert(name, value);
}
(object_info, encrypted, expected_sha256, headers)
}
async fn read_fixture_plaintext(encrypted: Vec<u8>, object_info: ObjectInfo, kms_key_b64: String) -> Result<Vec<u8>, String> {
async fn read_fixture_plaintext(
encrypted: Vec<u8>,
object_info: ObjectInfo,
headers: http::HeaderMap,
static_kms_key: Option<String>,
range: Option<HTTPRangeSpec>,
) -> Result<Vec<u8>, String> {
let object_size = object_info.size;
let full_object = range.is_none();
async_with_vars(
[
("__RUSTFS_SSE_SIMPLE_CMK", Some(kms_key_b64)),
("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
("RUSTFS_MINIO_STATIC_KMS_KEY", static_kms_key),
("__RUSTFS_SSE_SIMPLE_CMK", None::<String>),
],
async move {
let (mut reader, offset, length) = GetObjectReader::new(
Box::new(Cursor::new(encrypted)),
None,
&object_info,
&ObjectOptions::default(),
&http::HeaderMap::new(),
)
.await
.map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?;
let (mut reader, offset, length) =
GetObjectReader::new(Box::new(Cursor::new(encrypted)), range, &object_info, &ObjectOptions::default(), &headers)
.await
.map_err(|err| format!("construct GetObjectReader from MinIO raw fixture: {err:?}"))?;
if offset != 0 || length != object_size {
if full_object && (offset != 0 || length != object_size) {
return Err(format!("unexpected fixture range offset={offset} length={length} size={object_size}"));
}
@@ -164,6 +183,12 @@ async fn read_fixture_plaintext(encrypted: Vec<u8>, object_info: ObjectInfo, kms
}
async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, file_info: &FileInfo) -> Vec<u8> {
let erasure = Erasure::new_with_options(
file_info.erasure.data_blocks,
file_info.erasure.parity_blocks,
file_info.erasure.block_size,
file_info.uses_legacy_checksum,
);
let mut disks = Vec::with_capacity(file_info.erasure.distribution.len());
for disk_number in 1..=file_info.erasure.distribution.len() {
let disk_root = case_dir.join("backend").join(format!("disk{disk_number}"));
@@ -198,8 +223,13 @@ async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, fil
let mut encrypted = Vec::new();
for part in &file_info.parts {
let checksum_info = file_info.erasure.get_checksum_info(part.number);
let checksum_algorithm = if file_info.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S {
HashAlgorithm::HighwayHash256SLegacy
} else {
checksum_info.algorithm.clone()
};
let path = format!("{}/{}/part.{}", manifest.object, data_dir, part.number);
let shard_read_len = file_info.erasure.shard_file_size(part.size as i64);
let shard_read_len = erasure.shard_file_size(part.size as i64);
let mut readers = Vec::with_capacity(disks.len());
for (idx, disk) in disk_order.iter().enumerate() {
let reader = create_bitrot_reader(
@@ -210,7 +240,7 @@ async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, fil
0,
shard_read_len as usize,
file_info.erasure.shard_size(),
checksum_info.algorithm.clone(),
checksum_algorithm.clone(),
false,
false,
)
@@ -219,11 +249,6 @@ async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, fil
readers.push(reader);
}
let erasure = Erasure::new(
file_info.erasure.data_blocks,
file_info.erasure.parity_blocks,
file_info.erasure.block_size,
);
let mut writer = VecAsyncWriter::default();
let (written, err) = erasure.decode(&mut writer, readers, 0, part.size, part.size).await;
if let Some(err) = err {
@@ -244,6 +269,12 @@ async fn reads_minio_generated_sse_s3_multipart_fixture() {
assert_fixture_round_trip("sse-s3-multipart-8m", 8 * 1024 * 1024).await;
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_s3_singlepart_fixture() {
assert_fixture_round_trip("sse-s3-singlepart-64k", 64 * 1024).await;
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_kms_multipart_fixture() {
@@ -252,11 +283,47 @@ async fn reads_minio_generated_sse_kms_multipart_fixture() {
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key() {
let (object_info, encrypted, _) = load_fixture_reader_input("sse-s3-multipart-8m").await;
let wrong_key_b64 = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=".to_string();
async fn reads_minio_generated_sse_kms_singlepart_fixture() {
assert_fixture_round_trip("sse-kms-singlepart-64k", 64 * 1024).await;
}
let result = read_fixture_plaintext(encrypted, object_info, wrong_key_b64).await;
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_c_multipart_fixture() {
assert_fixture_round_trip("sse-c-multipart-8m", 8 * 1024 * 1024).await;
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_c_singlepart_fixture() {
assert_fixture_round_trip("sse-c-singlepart-64k", 64 * 1024).await;
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_s3_range_fixture() {
assert_fixture_range_round_trip("sse-s3-multipart-8m").await;
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_kms_range_fixture() {
assert_fixture_range_round_trip("sse-kms-multipart-8m").await;
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_c_range_fixture() {
assert_fixture_range_round_trip("sse-c-multipart-8m").await;
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key() {
let (object_info, encrypted, _, headers) = load_fixture_reader_input("sse-s3-multipart-8m").await;
let wrong_key_b64 = "minio-default-key:AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=".to_string();
let result = read_fixture_plaintext(encrypted, object_info, headers, Some(wrong_key_b64), None).await;
assert!(result.is_err(), "wrong KMS key must fail closed");
}
@@ -264,22 +331,16 @@ async fn rejects_minio_generated_sse_s3_fixture_with_wrong_kms_key() {
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext() {
let (object_info, mut encrypted, expected_sha256) = load_fixture_reader_input("sse-s3-multipart-8m").await;
let (object_info, mut encrypted, _, headers) = load_fixture_reader_input("sse-s3-multipart-8m").await;
encrypted.truncate(encrypted.len() / 2);
let result = read_fixture_plaintext(encrypted, object_info, minio_static_kms_key_b64()).await;
let result = read_fixture_plaintext(encrypted, object_info, headers, Some(minio_static_kms_key()), None).await;
if let Ok(plaintext) = result {
assert_ne!(
sha256_hex(&plaintext),
expected_sha256,
"truncated ciphertext must not restore the original plaintext"
);
}
assert!(result.is_err(), "truncated ciphertext must return an explicit read error");
}
async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
let (object_info, encrypted, expected_sha256) = load_fixture_reader_input(case_id).await;
let (object_info, encrypted, expected_sha256, headers) = load_fixture_reader_input(case_id).await;
// `ObjectInfo.size` is the on-disk size. For SSE objects that is the
// DARE-encrypted size (plaintext + 32 bytes per 64 KiB block), which is
// deliberately larger than the logical object size. The size a client sees
@@ -287,9 +348,9 @@ async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
// `decrypted_size()`/`get_actual_size()`, so assert against that — the raw
// `size` field would never equal the plaintext length for encrypted objects.
let decrypted_size = object_info.decrypted_size().expect("decrypted size from MinIO metadata");
let kms_key_b64 = minio_static_kms_key_b64();
let kms_key = minio_static_kms_key();
let plaintext = read_fixture_plaintext(encrypted, object_info, kms_key_b64)
let plaintext = read_fixture_plaintext(encrypted, object_info, headers, Some(kms_key), None)
.await
.expect("fixture must restore with the configured KMS key");
@@ -297,3 +358,59 @@ async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
assert_eq!(plaintext.len(), expected_size as usize);
assert_eq!(sha256_hex(&plaintext), expected_sha256);
}
async fn assert_fixture_range_round_trip(case_id: &str) {
const START: usize = 65_520;
const END: usize = 65_680;
let (object_info, encrypted, _, headers) = load_fixture_reader_input(case_id).await;
let kms_key = minio_static_kms_key();
let plaintext = read_fixture_plaintext(encrypted.clone(), object_info.clone(), headers.clone(), Some(kms_key.clone()), None)
.await
.expect("fixture full read must restore");
let ranged = read_fixture_plaintext(
encrypted,
object_info,
headers,
Some(kms_key),
Some(HTTPRangeSpec {
is_suffix_length: false,
start: START as i64,
end: END as i64,
}),
)
.await
.expect("fixture range read must restore");
assert_eq!(ranged, plaintext[START..=END]);
}
#[tokio::test]
#[ignore = "requires a fixture generated by the pinned RustFS beta.5 release"]
async fn reads_real_rustfs_beta5_sse_kms_fixture_through_production_reader() {
const CASE_ID: &str = "rustfs-beta5-sse-kms-singlepart-64k";
const KMS_KEY_ID: &str = "beta5-test-key";
let root = beta5_fixture_root();
let manager = rustfs_kms::init_global_kms_service_manager();
manager
.configure(
rustfs_kms::KmsConfig::local(root.join("kms"))
.with_default_key(KMS_KEY_ID.to_string())
.with_insecure_development_defaults(),
)
.await
.expect("configure production local KMS for beta.5 fixture");
manager.start().await.expect("start production local KMS for beta.5 fixture");
let (object_info, encrypted, expected_sha256, headers) =
load_fixture_reader_input_from(root.join("cases").join(CASE_ID)).await;
let expected_size = object_info.decrypted_size().expect("beta.5 encrypted object size");
let plaintext = read_fixture_plaintext(encrypted, object_info, headers, None, None)
.await
.expect("current production GET reader must decrypt the real beta.5 KMS fixture");
assert_eq!(expected_size, 64 * 1024);
assert_eq!(plaintext.len() as i64, expected_size);
assert_eq!(sha256_hex(&plaintext), expected_sha256);
}
+1
View File
@@ -36,6 +36,7 @@ pub(crate) mod legacy_bitrot_read {
}
pub(crate) mod minio_generated_read {
pub(crate) use super::storage_contracts::HTTPRangeSpec;
pub(crate) use super::{
DiskAPI, DiskOption, Endpoint, Erasure, GetObjectReader, ObjectInfo, ObjectOptions, create_bitrot_reader, new_disk,
};
+15 -51
View File
@@ -172,31 +172,21 @@ pub fn oidc_plugin_authn_metrics_snapshot() -> OidcPluginAuthnMetricsSnapshot {
OIDC_PLUGIN_AUTHN_METRICS.snapshot()
}
/// Header names whose values may carry OIDC secrets (client credentials, cookies,
/// bearer tokens). Their values are never emitted to logs, only their byte length.
const SENSITIVE_HEADER_NAMES: [&str; 4] = ["authorization", "proxy-authorization", "cookie", "set-cookie"];
fn is_sensitive_header(name: &str) -> bool {
SENSITIVE_HEADER_NAMES
.iter()
.any(|candidate| name.eq_ignore_ascii_case(candidate))
}
fn format_http_headers(headers: &http::HeaderMap) -> String {
headers
.iter()
.map(|(name, value)| {
if is_sensitive_header(name.as_str()) {
format!("{}=<redacted len={}>", name.as_str(), value.as_bytes().len())
} else {
let value = value.to_str().unwrap_or("<non-utf8>");
format!("{}={}", name.as_str(), value)
}
let value = value.to_str().unwrap_or("<non-utf8>");
format!("{}={}", name.as_str(), value)
})
.collect::<Vec<_>>()
.join("; ")
}
fn format_http_body(body: &[u8]) -> String {
String::from_utf8_lossy(body).into_owned()
}
#[derive(Debug, Default)]
struct TokenResponseBodyShape {
json_object: bool,
@@ -336,6 +326,7 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
let uri = parts.uri.to_string();
if tracing::enabled!(tracing::Level::DEBUG) {
let request_headers = format_http_headers(&parts.headers);
let request_body = format_http_body(&body);
debug!(
event = EVENT_OIDC_HTTP,
component = LOG_COMPONENT_IAM,
@@ -345,6 +336,7 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
uri = %uri,
request_headers = %request_headers,
request_body_len = body.len(),
request_body = %request_body,
"oidc outbound http"
);
}
@@ -395,6 +387,7 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
})?;
if tracing::enabled!(tracing::Level::DEBUG) {
let response_headers = format_http_headers(&headers);
let response_body = format_http_body(&body_bytes);
debug!(
event = EVENT_OIDC_HTTP,
component = LOG_COMPONENT_IAM,
@@ -407,6 +400,7 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
elapsed_ms,
response_headers = %response_headers,
response_body_len = body_bytes.len(),
response_body = %response_body,
"oidc outbound http"
);
}
@@ -820,6 +814,7 @@ impl OidcSys {
}
RequestTokenError::Parse(parse_err, body) => {
let shape = inspect_token_response_body(body);
let response_body = format_http_body(body);
error!(
event = EVENT_OIDC_DIAGNOSTICS,
component = LOG_COMPONENT_IAM,
@@ -842,11 +837,12 @@ impl OidcSys {
response_has_error = shape.has_error,
response_has_error_description = shape.has_error_description,
response_looks_like_html = shape.looks_like_html,
response_body = %response_body,
error = %e,
"oidc token exchange failed"
);
format!(
"token exchange failed: {e}: stage=token_response_parse_failed, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, parse_error_path={}, response_body_len={}, response_json_keys={}, response_has_id_token={}, response_has_error={}, response_looks_like_html={}",
"token exchange failed: {e}: stage=token_response_parse_failed, provider_id={}, config_url={}, issuer={}, token_endpoint={}, redirect_uri={}, client_id={}, parse_error_path={}, response_body_len={}, response_json_keys={}, response_has_id_token={}, response_has_error={}, response_looks_like_html={}, response_body={}",
session.provider_id,
config.config_url,
issuer,
@@ -858,7 +854,8 @@ impl OidcSys {
shape.json_keys,
shape.has_id_token,
shape.has_error,
shape.looks_like_html
shape.looks_like_html,
response_body
)
}
RequestTokenError::Other(message) => {
@@ -1852,39 +1849,6 @@ mod tests {
assert_eq!(extract_string_claim(&claims, "missing"), "");
}
#[test]
fn format_http_headers_redacts_sensitive_values() {
let mut headers = http::HeaderMap::new();
headers.insert(http::header::AUTHORIZATION, "Basic Y2xpZW50OnNlY3JldA==".parse().unwrap());
headers.insert(http::header::CONTENT_TYPE, "application/json".parse().unwrap());
headers.insert(http::header::COOKIE, "session=super-secret".parse().unwrap());
let rendered = format_http_headers(&headers);
// Sensitive header values never appear; only their length is emitted.
assert!(!rendered.contains("Y2xpZW50OnNlY3JldA=="), "authorization value leaked: {rendered}");
assert!(!rendered.contains("super-secret"), "cookie value leaked: {rendered}");
assert!(
rendered.contains("authorization=<redacted len="),
"expected redacted authorization: {rendered}"
);
assert!(rendered.contains("cookie=<redacted len="), "expected redacted cookie: {rendered}");
// Non-sensitive header values are preserved for diagnostics.
assert!(
rendered.contains("content-type=application/json"),
"content-type should be visible: {rendered}"
);
}
#[test]
fn is_sensitive_header_is_case_insensitive() {
assert!(is_sensitive_header("Authorization"));
assert!(is_sensitive_header("PROXY-AUTHORIZATION"));
assert!(is_sensitive_header("Set-Cookie"));
assert!(!is_sensitive_header("content-type"));
assert!(!is_sensitive_header("x-request-id"));
}
#[test]
fn test_extract_groups_claim_array() {
let mut claims = HashMap::new();
+1 -1
View File
@@ -56,7 +56,7 @@ moka = { workspace = true, features = ["future"] }
# Additional dependencies
md5 = { workspace = true }
arc-swap = { workspace = true }
rustfs-utils = { workspace = true }
rustfs-utils = { workspace = true, features = ["http"] }
rustfs-security-governance = { workspace = true }
# HTTP client for Vault
+35
View File
@@ -44,6 +44,23 @@ pub struct DataKeyEnvelope {
pub created_at: Zoned,
}
/// Return whether bytes contain a complete RustFS KMS data-key envelope.
pub fn is_data_key_envelope(ciphertext: &[u8]) -> bool {
const MAX_ENVELOPE_SIZE: usize = 64 * 1024;
if ciphertext.is_empty() || ciphertext.len() > MAX_ENVELOPE_SIZE {
return false;
}
serde_json::from_slice::<DataKeyEnvelope>(ciphertext).is_ok_and(|envelope| {
!envelope.key_id.trim().is_empty()
&& !envelope.master_key_id.trim().is_empty()
&& envelope.key_spec == "AES_256"
&& !envelope.encrypted_key.is_empty()
&& (envelope.nonce.is_empty() || envelope.nonce.len() == 12)
})
}
/// Trait for encrypting and decrypting data encryption keys (DEK)
///
/// This trait abstracts the encryption operations used to protect
@@ -293,6 +310,24 @@ mod tests {
assert_eq!(deserialized.key_id, envelope.key_id);
assert_eq!(deserialized.master_key_id, envelope.master_key_id);
assert_eq!(deserialized.encrypted_key, envelope.encrypted_key);
assert!(is_data_key_envelope(&serialized));
assert!(!is_data_key_envelope(b"not-a-kms-envelope"));
let mut boundary_envelope = serialized;
boundary_envelope.resize(64 * 1024, b' ');
assert!(is_data_key_envelope(&boundary_envelope));
boundary_envelope.push(b' ');
assert!(!is_data_key_envelope(&boundary_envelope));
let mut invalid = serde_json::to_value(&envelope).expect("Envelope should convert to JSON");
invalid["key_spec"] = serde_json::Value::String("AES_128".to_string());
assert!(!is_data_key_envelope(
&serde_json::to_vec(&invalid).expect("Invalid envelope should serialize")
));
invalid["key_spec"] = serde_json::Value::String("AES_256".to_string());
invalid["unknown"] = serde_json::Value::Bool(true);
assert!(is_data_key_envelope(
&serde_json::to_vec(&invalid).expect("Envelope with unknown field should serialize")
));
}
#[tokio::test]
+1 -1
View File
@@ -17,4 +17,4 @@
pub mod ciphers;
pub mod dek;
pub use dek::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
pub use dek::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material, is_data_key_envelope};
+5
View File
@@ -70,6 +70,7 @@ mod cache;
pub mod config;
mod encryption;
mod error;
mod managed_context;
pub mod manager;
pub mod service;
pub mod service_manager;
@@ -83,7 +84,11 @@ pub use api_types::{
TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse, UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
};
pub use config::*;
pub use encryption::is_data_key_envelope;
pub use error::{KmsError, Result};
pub use managed_context::{
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER, RUSTFS_ENCRYPTION_CONTEXT_HEADER, decode_managed_kms_context,
};
pub use manager::KmsManager;
pub use service::{DataKey, ObjectEncryptionService};
pub use service_manager::{
+110
View File
@@ -0,0 +1,110 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{KmsError, Result};
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use rustfs_utils::http::get_consistent_metadata_value;
use std::collections::HashMap;
pub const RUSTFS_ENCRYPTION_CONTEXT_HEADER: &str = "x-rustfs-encryption-context";
pub const MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER: &str = "X-Minio-Internal-Server-Side-Encryption-Context";
pub fn decode_managed_kms_context(metadata: &HashMap<String, String>) -> Result<Option<HashMap<String, String>>> {
let minio_context = consistent_value(metadata, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)?
.map(|context| {
let decoded = BASE64_STANDARD
.decode(context)
.map_err(|err| KmsError::serialization_error(format!("Failed to decode MinIO KMS context: {err}")))?;
serde_json::from_slice(&decoded)
.map_err(|err| KmsError::serialization_error(format!("Failed to parse MinIO KMS context: {err}")))
})
.transpose()?;
let rustfs_context = consistent_value(metadata, RUSTFS_ENCRYPTION_CONTEXT_HEADER)?
.map(|context| {
serde_json::from_str(context)
.map_err(|err| KmsError::serialization_error(format!("Failed to parse RustFS KMS context: {err}")))
})
.transpose()?;
match (minio_context, rustfs_context) {
(Some(minio), Some(rustfs)) if minio != rustfs => {
Err(KmsError::context_mismatch("Conflicting RustFS and MinIO KMS contexts"))
}
(Some(context), _) | (_, Some(context)) => Ok(Some(context)),
(None, None) => Ok(None),
}
}
fn consistent_value<'a>(metadata: &'a HashMap<String, String>, name: &str) -> Result<Option<&'a str>> {
get_consistent_metadata_value(metadata, name)
.map_err(|_| KmsError::validation_error(format!("Conflicting managed encryption metadata for {name}")))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn decode_context_accepts_compatible_headers_and_rejects_conflicts() {
let expected = HashMap::from([("tenant".to_string(), "alpha".to_string())]);
let metadata = HashMap::from([
(
RUSTFS_ENCRYPTION_CONTEXT_HEADER.to_string(),
serde_json::to_string(&expected).expect("RustFS KMS context should serialize"),
),
(
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(),
BASE64_STANDARD.encode(serde_json::to_vec(&expected).expect("MinIO KMS context should serialize")),
),
]);
assert_eq!(
decode_managed_kms_context(&metadata).expect("matching KMS contexts should parse"),
Some(expected.clone())
);
assert_eq!(
decode_managed_kms_context(&HashMap::from([(
RUSTFS_ENCRYPTION_CONTEXT_HEADER.to_string(),
serde_json::to_string(&expected).expect("legacy RustFS KMS context should serialize"),
)]))
.expect("legacy RustFS KMS context should parse"),
Some(expected)
);
let conflicting = HashMap::from([
(
RUSTFS_ENCRYPTION_CONTEXT_HEADER.to_string(),
serde_json::to_string(&HashMap::from([("tenant", "alpha")])).expect("RustFS KMS context should serialize"),
),
(
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(),
BASE64_STANDARD.encode(
serde_json::to_vec(&HashMap::from([("tenant", "beta")])).expect("MinIO KMS context should serialize"),
),
),
]);
assert!(decode_managed_kms_context(&conflicting).is_err());
assert!(
decode_managed_kms_context(&HashMap::from([(
MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(),
"not-base64".to_string(),
)]))
.is_err()
);
assert!(
decode_managed_kms_context(&HashMap::from([(RUSTFS_ENCRYPTION_CONTEXT_HEADER.to_string(), "not-json".to_string(),)]))
.is_err()
);
}
}
+1 -195
View File
@@ -1156,50 +1156,10 @@ pub struct SRStateEditReq {
pub struct ResyncBucketStatus {
#[serde(default)]
pub bucket: String,
#[serde(rename = "targetArn", default, skip_serializing_if = "String::is_empty")]
pub target_arn: String,
#[serde(default)]
pub status: String,
#[serde(rename = "errorDetail", skip_serializing_if = "String::is_empty", default)]
pub err_detail: String,
#[serde(
rename = "createdAt",
default,
with = "time::serde::rfc3339::option",
skip_serializing_if = "Option::is_none"
)]
pub created_at: Option<OffsetDateTime>,
#[serde(
rename = "startedAt",
default,
with = "time::serde::rfc3339::option",
skip_serializing_if = "Option::is_none"
)]
pub started_at: Option<OffsetDateTime>,
#[serde(
rename = "updatedAt",
default,
with = "time::serde::rfc3339::option",
skip_serializing_if = "Option::is_none"
)]
pub updated_at: Option<OffsetDateTime>,
#[serde(
rename = "completedAt",
default,
with = "time::serde::rfc3339::option",
skip_serializing_if = "Option::is_none"
)]
pub completed_at: Option<OffsetDateTime>,
#[serde(default, skip_serializing_if = "is_zero_u64")]
pub generation: u64,
#[serde(rename = "replicatedObjects", default, skip_serializing_if = "is_zero_u64")]
pub replicated_objects: u64,
#[serde(rename = "replicatedBytes", default, skip_serializing_if = "is_zero_u64")]
pub replicated_bytes: u64,
#[serde(rename = "failedObjects", default, skip_serializing_if = "is_zero_u64")]
pub failed_objects: u64,
#[serde(rename = "failedBytes", default, skip_serializing_if = "is_zero_u64")]
pub failed_bytes: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -1210,74 +1170,10 @@ pub struct SRResyncOpStatus {
pub resync_id: String,
#[serde(default)]
pub status: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub state: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub buckets: Vec<ResyncBucketStatus>,
#[serde(rename = "errorDetail", skip_serializing_if = "String::is_empty", default)]
pub err_detail: String,
#[serde(
rename = "createdAt",
default,
with = "time::serde::rfc3339::option",
skip_serializing_if = "Option::is_none"
)]
pub created_at: Option<OffsetDateTime>,
#[serde(
rename = "startedAt",
default,
with = "time::serde::rfc3339::option",
skip_serializing_if = "Option::is_none"
)]
pub started_at: Option<OffsetDateTime>,
#[serde(
rename = "updatedAt",
default,
with = "time::serde::rfc3339::option",
skip_serializing_if = "Option::is_none"
)]
pub updated_at: Option<OffsetDateTime>,
#[serde(
rename = "completedAt",
default,
with = "time::serde::rfc3339::option",
skip_serializing_if = "Option::is_none"
)]
pub completed_at: Option<OffsetDateTime>,
#[serde(default, skip_serializing_if = "is_zero_u64")]
pub generation: u64,
#[serde(rename = "totalBuckets", default, skip_serializing_if = "is_zero_u64")]
pub total_buckets: u64,
#[serde(rename = "pendingBuckets", default, skip_serializing_if = "is_zero_u64")]
pub pending_buckets: u64,
#[serde(rename = "runningBuckets", default, skip_serializing_if = "is_zero_u64")]
pub running_buckets: u64,
#[serde(rename = "completedBuckets", default, skip_serializing_if = "is_zero_u64")]
pub completed_buckets: u64,
#[serde(rename = "failedBuckets", default, skip_serializing_if = "is_zero_u64")]
pub failed_buckets: u64,
#[serde(rename = "canceledBuckets", default, skip_serializing_if = "is_zero_u64")]
pub canceled_buckets: u64,
#[serde(rename = "replicatedObjects", default, skip_serializing_if = "is_zero_u64")]
pub replicated_objects: u64,
#[serde(rename = "replicatedBytes", default, skip_serializing_if = "is_zero_u64")]
pub replicated_bytes: u64,
#[serde(rename = "failedObjects", default, skip_serializing_if = "is_zero_u64")]
pub failed_objects: u64,
#[serde(rename = "failedBytes", default, skip_serializing_if = "is_zero_u64")]
pub failed_bytes: u64,
#[serde(default, skip_serializing_if = "is_false")]
pub truncated: bool,
#[serde(rename = "nextContinuationToken", default, skip_serializing_if = "String::is_empty")]
pub next_continuation_token: String,
}
fn is_zero_u64(value: &u64) -> bool {
*value == 0
}
fn is_false(value: &bool) -> bool {
!*value
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
@@ -1306,7 +1202,7 @@ pub struct SiteNetPerfResult {
#[cfg(test)]
mod tests {
use super::{PeerInfo, PeerSite, SRResyncOpStatus};
use super::{PeerInfo, PeerSite};
use serde_json::{Value, json};
const TEST_CA_CERT: &str = "-----BEGIN CERTIFICATE-----\ntest-ca\n-----END CERTIFICATE-----";
@@ -1406,94 +1302,4 @@ mod tests {
assert!(peer_debug.contains("skip_tls_verify: false"));
assert!(peer_debug.contains("has_custom_ca: true"));
}
#[test]
fn resync_status_legacy_json_defaults_new_lifecycle_fields() {
let legacy_json = json!({
"op": "start",
"id": "resync-1",
"status": "success",
"buckets": [{
"bucket": "photos",
"status": "success"
}]
});
let status: SRResyncOpStatus =
serde_json::from_value(legacy_json.clone()).expect("legacy resync status should deserialize");
assert_eq!(status.generation, 0);
assert!(status.state.is_empty());
assert!(status.created_at.is_none());
assert_eq!(status.total_buckets, 0);
assert_eq!(status.replicated_objects, 0);
assert!(!status.truncated);
assert!(status.next_continuation_token.is_empty());
assert!(status.buckets[0].created_at.is_none());
assert!(status.buckets[0].target_arn.is_empty());
assert_eq!(status.buckets[0].generation, 0);
assert_eq!(status.buckets[0].replicated_bytes, 0);
assert_eq!(serde_json::to_value(status).expect("legacy resync status should serialize"), legacy_json);
}
#[test]
fn resync_status_lifecycle_fields_round_trip_with_exact_json_names() {
let status_json = json!({
"op": "status",
"id": "resync-2",
"status": "success",
"state": "running",
"createdAt": "2026-07-22T01:00:00Z",
"startedAt": "2026-07-22T01:00:01Z",
"updatedAt": "2026-07-22T01:01:00Z",
"completedAt": "2026-07-22T01:02:00Z",
"generation": 7,
"totalBuckets": 6,
"pendingBuckets": 1,
"runningBuckets": 1,
"completedBuckets": 1,
"failedBuckets": 1,
"canceledBuckets": 2,
"replicatedObjects": 12,
"replicatedBytes": 4096,
"failedObjects": 3,
"failedBytes": 512,
"truncated": true,
"nextContinuationToken": "bucket-page-2",
"buckets": [{
"bucket": "photos",
"targetArn": "arn:rustfs:replication::peer-a:photos",
"status": "failed",
"errorDetail": "target unavailable",
"createdAt": "2026-07-22T01:00:00Z",
"startedAt": "2026-07-22T01:00:01Z",
"updatedAt": "2026-07-22T01:01:00Z",
"completedAt": "2026-07-22T01:02:00Z",
"generation": 7,
"replicatedObjects": 12,
"replicatedBytes": 4096,
"failedObjects": 3,
"failedBytes": 512
}]
});
let status: SRResyncOpStatus =
serde_json::from_value(status_json.clone()).expect("expanded resync status should deserialize");
assert_eq!(status.generation, 7);
assert_eq!(status.state, "running");
assert_eq!(status.total_buckets, 6);
assert_eq!(status.completed_buckets, 1);
assert_eq!(status.replicated_bytes, 4096);
assert!(status.truncated);
assert_eq!(status.next_continuation_token, "bucket-page-2");
assert_eq!(status.buckets[0].generation, 7);
assert_eq!(status.buckets[0].target_arn, "arn:rustfs:replication::peer-a:photos");
assert_eq!(status.buckets[0].failed_objects, 3);
assert!(status.buckets[0].completed_at.is_some());
assert_eq!(
serde_json::to_value(status).expect("expanded resync status should serialize"),
status_json
);
}
}
+1 -9
View File
@@ -154,7 +154,7 @@ impl TargetReplicationResyncStatus {
}
fn marshal_wire_msg(&self, wr: &mut Vec<u8>) -> Result<()> {
rmp::encode::write_map_len(wr, 12)?;
rmp::encode::write_map_len(wr, 11)?;
rmp::encode::write_str(wr, "st")?;
write_msgp_time(wr, wire_time_or_default(self.start_time))?;
rmp::encode::write_str(wr, "lst")?;
@@ -177,8 +177,6 @@ impl TargetReplicationResyncStatus {
rmp::encode::write_str(wr, &self.bucket)?;
rmp::encode::write_str(wr, "obj")?;
rmp::encode::write_str(wr, &self.object)?;
rmp::encode::write_str(wr, "err")?;
rmp::encode::write_str(wr, self.error.as_deref().unwrap_or_default())?;
Ok(())
}
@@ -204,10 +202,6 @@ impl TargetReplicationResyncStatus {
"rrc" => out.replicated_count = rmp::decode::read_int(rd)?,
"bkt" => out.bucket = read_msgp_str(rd)?,
"obj" => out.object = read_msgp_str(rd)?,
"err" => {
let error = read_msgp_str(rd)?;
out.error = (!error.is_empty()).then_some(error);
}
_ => skip_msgp_value(rd)?,
}
}
@@ -629,7 +623,6 @@ mod tests {
bucket: "bucket-a".to_string(),
object: "object-a".to_string(),
replicated_count: 7,
error: Some("durable failure".to_string()),
..Default::default()
},
);
@@ -641,7 +634,6 @@ mod tests {
assert_eq!(got.targets_map["arn:replication:a"].resync_id, "rid-1");
assert_eq!(got.targets_map["arn:replication:a"].resync_status, ResyncStatusType::ResyncStarted);
assert_eq!(got.targets_map["arn:replication:a"].replicated_count, 7);
assert_eq!(got.targets_map["arn:replication:a"].error.as_deref(), Some("durable failure"));
}
#[test]
@@ -136,7 +136,7 @@ tests read):
# Pass case ids to override, or "all" for the full default matrix.
./capture_via_docker.sh
RUSTFS_MINIO_STATIC_KMS_KEY_B64=IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= \
RUSTFS_MINIO_STATIC_KMS_KEY=minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= \
cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
```
@@ -145,6 +145,28 @@ This is exactly what the nightly `minio-interop` GitHub Actions workflow runs
SSE-C cases still need the host-`minio` + TLS path above; the Docker helper
targets the SSE-S3 / SSE-KMS multipart cases the interop tests assert on.
## RustFS beta.5 KMS Compatibility Fixture
The same CI gate also downloads the pinned RustFS `1.0.0-beta.5` release,
verifies its published archive SHA-256, starts it with the production local KMS
backend, writes a real SSE-KMS object, and exports the resulting four-disk
backend plus its one-time KMS key directory:
```bash
uv run python ./capture_rustfs_beta5.py
RUSTFS_BETA5_FIXTURE_ROOT=../fixtures/rustfs-beta5-generated \
cargo test -p rustfs-ecstore --features rio-v2 \
--test minio_generated_read_test \
reads_real_rustfs_beta5_sse_kms_fixture_through_production_reader \
-- --ignored
```
Outside Linux x86_64, pass `--rustfs-binary` with the matching beta.5 binary.
The generated fixture and KMS key stay under the ignored fixture root and are
recreated for every CI run; neither key material nor plaintext keys are logged
or committed.
## Capture Guidance
For each case, preserve these inputs when possible:
@@ -0,0 +1,224 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import base64
import hashlib
import json
import os
import platform
import secrets
import shutil
import stat
import subprocess
import tempfile
import time
import urllib.error
import urllib.request
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlparse
from lab import (
FixtureCase,
LabPaths,
S3Client,
build_payload_file,
head_case,
stop_process,
store_case_artifacts,
upload_case,
wait_for_s3_ready,
)
RELEASE = "1.0.0-beta.5"
CASE_ID = "rustfs-beta5-sse-kms-singlepart-64k"
KMS_KEY_ID = "beta5-test-key"
LINUX_X86_64_ASSET = "rustfs-linux-x86_64-gnu-v1.0.0-beta.5.zip"
LINUX_X86_64_SHA256 = "73529e732adc3c2c5c78c7f2c2e4e331a20e5bee1e2db341ee7c762299e4b327"
DEFAULT_ROOT = Path(__file__).resolve().parents[1] / "fixtures" / "rustfs-beta5-generated"
def release_asset_url(asset: str) -> str:
return f"https://github.com/rustfs/rustfs/releases/download/{RELEASE}/{asset}"
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def resolve_binary(explicit_binary: Path | None, workdir: Path) -> Path:
if explicit_binary is not None:
binary = explicit_binary.resolve()
if not binary.is_file():
raise FileNotFoundError(f"RustFS beta.5 binary not found: {binary}")
return binary
if platform.system() != "Linux" or platform.machine() not in {"x86_64", "AMD64"}:
raise RuntimeError("--rustfs-binary is required outside Linux x86_64")
archive = workdir / LINUX_X86_64_ASSET
urllib.request.urlretrieve(release_asset_url(LINUX_X86_64_ASSET), archive)
actual = sha256_file(archive)
if actual != LINUX_X86_64_SHA256:
raise RuntimeError(f"RustFS beta.5 archive SHA-256 mismatch: expected {LINUX_X86_64_SHA256}, got {actual}")
with zipfile.ZipFile(archive) as bundle:
bundle.extract("rustfs", workdir)
binary = workdir / "rustfs"
binary.chmod(binary.stat().st_mode | stat.S_IXUSR)
return binary
def write_local_kms_key(key_dir: Path) -> None:
key_dir.mkdir(parents=True, mode=0o700)
key_path = key_dir / f"{KMS_KEY_ID}.key"
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
payload = {
"key_id": KMS_KEY_ID,
"version": 1,
"algorithm": "AES_256",
"usage": "EncryptDecrypt",
"status": "Active",
"description": None,
"metadata": {},
"created_at": now,
"rotated_at": None,
"created_by": "fixture-lab",
"encrypted_key_material": base64.b64encode(secrets.token_bytes(32)).decode("ascii"),
"nonce": [],
}
key_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
key_path.chmod(0o600)
def wait_for_process_s3(client: S3Client, process: subprocess.Popen[str], timeout_seconds: int) -> None:
deadline = time.time() + timeout_seconds
while time.time() < deadline:
if process.poll() is not None:
raise RuntimeError("RustFS beta.5 exited before its S3 API became ready")
try:
client.list_buckets()
return
except (RuntimeError, urllib.error.URLError):
time.sleep(1)
wait_for_s3_ready(client, 1)
def capture(args: argparse.Namespace) -> Path:
root = args.root.resolve()
if root.exists():
shutil.rmtree(root)
root.mkdir(parents=True)
with tempfile.TemporaryDirectory(prefix="rustfs-beta5-fixture-") as temporary:
workdir = Path(temporary)
binary = resolve_binary(args.rustfs_binary, workdir)
key_dir = workdir / "kms"
write_local_kms_key(key_dir)
disks = [workdir / f"disk{index}" for index in range(1, 5)]
for disk in disks:
disk.mkdir()
endpoint = args.endpoint
parsed = urlparse(endpoint)
if parsed.scheme != "http" or not parsed.netloc:
raise ValueError("beta.5 fixture endpoint must be an http:// URL")
command = [
str(binary),
"server",
"--address",
parsed.netloc,
"--access-key",
"minioadmin",
"--secret-key",
"minioadmin",
"--kms-enable",
"--kms-backend",
"local",
"--kms-key-dir",
str(key_dir),
"--kms-default-key-id",
KMS_KEY_ID,
*(str(disk) for disk in disks),
]
environment = os.environ.copy()
environment["RUSTFS_UNSAFE_BYPASS_DISK_CHECK"] = "true"
server_log_path = workdir / "server.log"
case = FixtureCase(
case_id=CASE_ID,
bucket="demo",
object_name="dir/object.bin",
encryption="SSE-KMS",
size_bytes=64 * 1024,
multipart=False,
kms_key_id=KMS_KEY_ID,
)
payload_file = workdir / "payload.bin"
plaintext_sha256 = build_payload_file(payload_file, case.size_bytes)
with server_log_path.open("w", encoding="utf-8") as server_log:
process = subprocess.Popen(
command,
cwd=str(workdir),
env=environment,
stdout=server_log,
stderr=subprocess.STDOUT,
text=True,
)
try:
client = S3Client(endpoint)
wait_for_process_s3(client, process, args.timeout_seconds)
client.create_bucket(case.bucket)
request_payload = upload_case(client, case, payload_file)
head_payload = head_case(client, case)
finally:
stop_process(process)
export_root = workdir / "export"
export_root.mkdir()
for index, disk in enumerate(disks, start=1):
shutil.copytree(disk, export_root / f"disk{index}")
shutil.copytree(key_dir, root / "kms")
return store_case_artifacts(
paths=LabPaths(root=root, cases=root / "cases"),
case_id=case.case_id,
bucket=case.bucket,
object_name=case.object_name,
source_tree=export_root,
version_id=head_payload.get("VersionId"),
request_payload=request_payload,
head_payload=head_payload,
plaintext_sha256=plaintext_sha256,
notes="Captured from the pinned RustFS 1.0.0-beta.5 release with the production local KMS backend.",
capture_payload={
"release": RELEASE,
"binary_sha256": sha256_file(binary),
"disk_count": len(disks),
"kms_key_id": KMS_KEY_ID,
},
)
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Capture a real RustFS beta.5 SSE-KMS fixture")
parser.add_argument("--root", type=Path, default=DEFAULT_ROOT)
parser.add_argument("--rustfs-binary", type=Path)
parser.add_argument("--endpoint", default="http://127.0.0.1:19011")
parser.add_argument("--timeout-seconds", type=int, default=90)
return parser
def main() -> int:
case_dir = capture(build_parser().parse_args())
print(case_dir)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -2,6 +2,7 @@ from __future__ import annotations
import importlib.util
import base64
import json
import sys
import tempfile
import unittest
@@ -21,6 +22,21 @@ def load_lab_module():
lab = load_lab_module()
sys.modules["lab"] = lab
def load_beta5_module():
module_path = Path(__file__).with_name("capture_rustfs_beta5.py")
spec = importlib.util.spec_from_file_location("rustfs_beta5_fixture_capture", module_path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
beta5 = load_beta5_module()
class DiscoverMinioLauncherTests(unittest.TestCase):
@@ -203,5 +219,31 @@ class KmsSecretKeyTests(unittest.TestCase):
)
class RustfsBeta5CaptureTests(unittest.TestCase):
def test_explicit_beta5_binary_is_used_without_download(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
binary = Path(temp_dir) / "rustfs"
binary.write_bytes(b"beta5")
resolved = beta5.resolve_binary(binary, Path(temp_dir))
self.assertEqual(resolved, binary.resolve())
def test_local_kms_key_has_beta5_compatible_shape_and_permissions(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
key_dir = Path(temp_dir) / "kms"
beta5.write_local_kms_key(key_dir)
key_path = key_dir / f"{beta5.KMS_KEY_ID}.key"
payload = json.loads(key_path.read_text(encoding="utf-8"))
self.assertEqual(payload["key_id"], beta5.KMS_KEY_ID)
self.assertEqual(payload["algorithm"], "AES_256")
self.assertEqual(payload["usage"], "EncryptDecrypt")
self.assertEqual(len(base64.b64decode(payload["encrypted_key_material"])), 32)
self.assertEqual(payload["nonce"], [])
self.assertEqual(key_path.stat().st_mode & 0o777, 0o600)
if __name__ == "__main__":
unittest.main()
+1
View File
@@ -2514,6 +2514,7 @@ mod tests {
stream: Box::new(Cursor::new(data)),
object_info: ObjectInfo::default(),
buffered_body: None,
resolved_sse: None,
body_source: Default::default(),
})
}
-9
View File
@@ -22,7 +22,6 @@ pub enum CapabilityState {
Supported,
Unsupported,
Disabled,
#[serde(other)]
#[default]
Unknown,
}
@@ -134,12 +133,4 @@ mod tests {
assert_eq!(decoded.1.reason.as_deref(), Some("target does not expose profiler"));
assert!(!decoded.1.state.is_supported());
}
#[test]
fn future_capability_state_deserializes_as_unknown() {
let status: CapabilityStatus = serde_json::from_str(r#"{"state":"future_state"}"#)
.expect("future capability states should preserve conservative compatibility");
assert_eq!(status.state, CapabilityState::Unknown);
}
}
+4 -14
View File
@@ -23,7 +23,7 @@ use rustfs_config::{
NATS_TLS_CLIENT_KEY, NATS_TOKEN, NATS_USERNAME, PULSAR_AUTH_TOKEN, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_TLS_CA,
PULSAR_TOPIC, PULSAR_USERNAME,
};
use rustfs_utils::egress::OutboundPolicy;
use rustfs_utils::egress::validate_outbound_url;
use std::collections::HashSet;
use std::path::Path;
use std::str::FromStr;
@@ -215,20 +215,16 @@ pub(super) fn validate_pulsar_broker_config(broker: &str, config: &KVS, default_
}
pub(super) fn parse_url(value: &str, field_label: &str) -> Result<Url, TargetError> {
Url::parse(value).map_err(|e| TargetError::Configuration(format!("Invalid {field_label}: {e}")))
Url::parse(value).map_err(|e| TargetError::Configuration(format!("Invalid {field_label}: {e} (value: '{value}')")))
}
pub(super) fn validate_outbound_http_url(value: &Url, field_label: &str) -> Result<(), TargetError> {
let policy =
OutboundPolicy::from_env_cached().map_err(|err| TargetError::Configuration(format!("invalid outbound policy: {err}")))?;
policy
.validate_url(value)
.map_err(|e| TargetError::Configuration(format!("{field_label} is not allowed: {e}")))
validate_outbound_url(value).map_err(|e| TargetError::Configuration(format!("{field_label} is not allowed: {e}")))
}
#[cfg(test)]
mod tests {
use super::{parse_jetstream_enable, parse_url, validate_nats_server_config, validate_pulsar_broker_config};
use super::{parse_jetstream_enable, validate_nats_server_config, validate_pulsar_broker_config};
use async_nats::ServerAddr;
use rustfs_config::server_config::KVS;
use rustfs_config::{
@@ -242,12 +238,6 @@ mod tests {
ServerAddr::from_str("nats://127.0.0.1:4222").expect("valid nats address")
}
#[test]
fn parse_url_error_does_not_echo_the_configured_value() {
let err = parse_url("not a URL containing secret-token", "endpoint URL").expect_err("invalid URL should fail");
assert!(!err.to_string().contains("secret-token"));
}
// Absolute on Linux, macOS, and Windows. temp_dir needs no filesystem to exist for a
// validation-only test, and Path::is_absolute stays true across platforms.
fn nats_queue_dir() -> String {
+2 -36
View File
@@ -84,23 +84,7 @@ fn redact_target_field_value(field_name: &str, value: &str) -> String {
return crate::target::mysql::redact_mysql_dsn(value);
}
if is_sensitive_target_field(field_name) {
return crate::target::REDACTED_SECRET.to_string();
}
if field_name.eq_ignore_ascii_case(rustfs_config::WEBHOOK_ENDPOINT)
|| field_name.eq_ignore_ascii_case(rustfs_config::AMQP_URL)
{
return url::Url::parse(value)
.ok()
.and_then(|endpoint| {
let host = match endpoint.host()? {
url::Host::Domain(host) => host.to_string(),
url::Host::Ipv4(host) => host.to_string(),
url::Host::Ipv6(host) => format!("[{host}]"),
};
let port = endpoint.port().map(|port| format!(":{port}")).unwrap_or_default();
Some(format!("{}://{host}{port}", endpoint.scheme()))
})
.unwrap_or_else(|| crate::target::REDACTED_SECRET.to_string());
return "***redacted***".to_string();
}
value.to_string()
}
@@ -648,24 +632,6 @@ mod tests {
assert_eq!(redact_target_field_value("queue_limit", "1000"), "1000");
}
#[test]
fn redact_target_field_value_strips_endpoint_path_and_query() {
assert_eq!(
redact_target_field_value("endpoint", "https://example.com/private/hook?token=secret"),
"https://example.com"
);
assert_eq!(redact_target_field_value("endpoint", "not a URL with secret"), "***redacted***");
}
#[test]
fn redact_target_field_value_strips_url_credentials() {
assert_eq!(
redact_target_field_value("url", "amqps://user:secret@broker.example/vhost"),
"amqps://broker.example"
);
assert_eq!(redact_target_field_value("url", "not a URL with secret"), "***redacted***");
}
#[test]
fn redact_dsn_string_partial_redaction() {
let dsn = "rustfs:secret123@tcp(mysql.example.com:3306)/rustfs_events";
@@ -705,7 +671,7 @@ mod tests {
assert_eq!(
redacted,
vec![
("endpoint".to_string(), "https://example.com".to_string()),
("endpoint".to_string(), "https://example.com/hook".to_string()),
("password".to_string(), "***redacted***".to_string()),
("client_key".to_string(), "***redacted***".to_string()),
("auth_token".to_string(), "***redacted***".to_string()),
+15 -225
View File
@@ -32,7 +32,7 @@ use async_trait::async_trait;
use parking_lot::Mutex;
use reqwest::{Client, StatusCode, Url};
use rustfs_tls_runtime::load_cert_bundle_der_bytes;
use rustfs_utils::egress::OutboundPolicy;
use rustfs_utils::egress::validate_outbound_url;
use std::{
error::Error as StdError,
fmt,
@@ -166,8 +166,7 @@ impl WebhookArgs {
if self.endpoint.as_str().is_empty() {
return Err(TargetError::Configuration("endpoint empty".to_string()));
}
outbound_policy()?
.validate_url(&self.endpoint)
validate_outbound_url(&self.endpoint)
.map_err(|err| TargetError::Configuration(format!("webhook endpoint is not allowed: {err}")))?;
if !self.queue_dir.is_empty() {
@@ -248,16 +247,8 @@ where
None
};
let http_client = if args.enable {
Self::build_http_client(&args)?
} else {
Client::builder()
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.build()
.map_err(|e| TargetError::Configuration(format!("Failed to build disabled webhook HTTP client: {e}")))?
};
let http_client = Arc::new(Mutex::new(http_client));
// Build HTTP client using the helper function
let http_client = Arc::new(Mutex::new(Self::build_http_client(&args)?));
let queue_store = open_target_queue_store(
&args.queue_dir,
@@ -294,19 +285,7 @@ where
}
fn build_http_client(args: &WebhookArgs) -> Result<Client, TargetError> {
let resolver = outbound_policy()?
.resolver_for(&args.endpoint)
.map_err(|err| TargetError::Configuration(format!("webhook endpoint is not allowed: {err}")))?;
Self::build_http_client_with_resolver(args, resolver)
}
fn build_http_client_with_resolver(
args: &WebhookArgs,
resolver: impl reqwest::dns::Resolve + 'static,
) -> Result<Client, TargetError> {
let mut client_builder = Client::builder()
.no_proxy()
.dns_resolver(resolver)
.timeout(Duration::from_secs(30))
// SSRF hardening (backlog#974): never follow HTTP redirects on webhook delivery.
// reqwest follows up to 10 redirects by default, which lets a malicious or
@@ -315,6 +294,11 @@ where
// bypassing the outbound-endpoint validation performed on the configured URL.
.redirect(reqwest::redirect::Policy::none())
.user_agent(crate::get_user_agent(crate::ServiceType::Basis));
#[cfg(test)]
{
client_builder = client_builder.no_proxy();
}
// 1. Configure server certificate verification
if args.skip_tls_verify {
// DANGEROUS: For testing only, skip all certificate verification
@@ -323,7 +307,6 @@ where
event = EVENT_WEBHOOK_TARGET_STATE,
component = LOG_COMPONENT_TARGETS,
subsystem = LOG_SUBSYSTEM_WEBHOOK,
endpoint_origin = %args.endpoint.origin().ascii_serialization(),
state = "tls_verification_skipped",
fallback = "danger_accept_invalid_certs",
"webhook target state"
@@ -553,10 +536,6 @@ where
}
}
fn outbound_policy() -> Result<&'static OutboundPolicy, TargetError> {
OutboundPolicy::from_env_cached().map_err(|err| TargetError::Configuration(format!("invalid outbound policy: {err}")))
}
#[async_trait]
impl<E> Target<E> for WebhookTarget<E>
where
@@ -776,61 +755,10 @@ where
mod tests {
use super::{WebhookArgs, WebhookTarget, classify_delivery_status, probe_health_url};
use crate::target::{REDACTED_SECRET, Target, TargetHealthReason, TargetHealthState, TargetType, decode_object_name};
use std::net::{IpAddr, SocketAddr};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use url::Url;
use url::form_urlencoded;
#[derive(Clone)]
struct StaticResolver(IpAddr);
impl reqwest::dns::Resolve for StaticResolver {
fn resolve(&self, _name: reqwest::dns::Name) -> reqwest::dns::Resolving {
let address = SocketAddr::new(self.0, 0);
Box::pin(async move { Ok(Box::new(std::iter::once(address)) as reqwest::dns::Addrs) })
}
}
#[derive(Clone)]
struct FailingResolver;
impl reqwest::dns::Resolve for FailingResolver {
fn resolve(&self, _name: reqwest::dns::Name) -> reqwest::dns::Resolving {
Box::pin(async { Err(std::io::Error::new(std::io::ErrorKind::NotFound, "test DNS failure").into()) })
}
}
#[derive(Clone, Default)]
struct CapturedLog(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
struct CapturedLogWriter(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
impl std::io::Write for CapturedLogWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.lock().expect("captured log lock").extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'writer> tracing_subscriber::fmt::MakeWriter<'writer> for CapturedLog {
type Writer = CapturedLogWriter;
fn make_writer(&'writer self) -> Self::Writer {
CapturedLogWriter(self.0.clone())
}
}
impl CapturedLog {
fn contents(&self) -> String {
String::from_utf8(self.0.lock().expect("captured log lock").clone()).expect("captured logs must be UTF-8")
}
}
fn base_args() -> WebhookArgs {
WebhookArgs {
enable: true,
@@ -882,103 +810,6 @@ mod tests {
assert!(rendered.contains("WebhookArgs"));
}
#[tokio::test]
async fn webhook_client_uses_the_supplied_connection_resolver() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind resolver test listener");
let address = listener.local_addr().expect("resolver test listener address");
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("accept resolved webhook request");
let mut request = [0_u8; 1024];
let read = stream.read(&mut request).await.expect("read webhook request");
assert!(String::from_utf8_lossy(&request[..read]).starts_with("GET /hook HTTP/1.1"));
stream
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.await
.expect("write webhook response");
});
let args = WebhookArgs {
endpoint: Url::parse(&format!("http://webhook.test:{}/hook", address.port())).expect("endpoint should parse"),
..base_args()
};
let client = WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(&args, StaticResolver(address.ip()))
.expect("webhook client should build");
let response = client
.get(args.endpoint)
.send()
.await
.expect("resolver should route request to test listener");
assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT);
server.await.expect("resolver test server should finish");
}
#[test]
fn webhook_client_ignores_environment_proxies_before_dns_filtering() {
const CHILD_ENV: &str = "RUSTFS_TEST_WEBHOOK_PROXY_CHILD";
const TARGET_URL_ENV: &str = "RUSTFS_TEST_WEBHOOK_PROXY_TARGET_URL";
const TARGET_ADDR_ENV: &str = "RUSTFS_TEST_WEBHOOK_PROXY_TARGET_ADDR";
if std::env::var_os(CHILD_ENV).is_some() {
let endpoint = Url::parse(&std::env::var(TARGET_URL_ENV).expect("child target URL")).expect("target URL");
let address = std::env::var(TARGET_ADDR_ENV)
.expect("child target address")
.parse::<SocketAddr>()
.expect("target address");
let args = WebhookArgs { endpoint, ..base_args() };
let client = WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(&args, StaticResolver(address.ip()))
.expect("webhook client should build");
tokio::runtime::Runtime::new().expect("child runtime").block_on(async {
let response = client
.get(args.endpoint)
.send()
.await
.expect("webhook request should bypass environment proxy");
assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT);
});
return;
}
use std::io::{Read, Write};
use std::net::TcpListener as StdTcpListener;
let target_listener = StdTcpListener::bind("127.0.0.1:0").expect("bind webhook target listener");
let target_address = target_listener.local_addr().expect("webhook target address");
let target = std::thread::spawn(move || {
let (mut stream, _) = target_listener.accept().expect("accept direct webhook request");
let mut request = [0_u8; 1024];
let read = stream.read(&mut request).expect("read direct webhook request");
assert!(String::from_utf8_lossy(&request[..read]).starts_with("GET /hook HTTP/1.1"));
stream
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.expect("write direct webhook response");
});
let proxy_listener = StdTcpListener::bind("127.0.0.1:0").expect("reserve refused proxy address");
let proxy_address = proxy_listener.local_addr().expect("proxy address");
drop(proxy_listener);
let target_url = format!("http://webhook.test:{}/hook", target_address.port());
let proxy_url = format!("http://{proxy_address}");
let output = std::process::Command::new(std::env::current_exe().expect("resolve current test executable"))
.arg("webhook_client_ignores_environment_proxies_before_dns_filtering")
.arg("--nocapture")
.env(CHILD_ENV, "1")
.env(TARGET_URL_ENV, target_url)
.env(TARGET_ADDR_ENV, target_address.to_string())
.env("HTTP_PROXY", &proxy_url)
.env("HTTPS_PROXY", &proxy_url)
.env("ALL_PROXY", &proxy_url)
.env("NO_PROXY", "")
.output()
.expect("run isolated proxy test child");
assert!(
output.status.success(),
"proxy test child failed: stdout={} stderr={}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
target.join().expect("direct webhook target should finish");
}
#[test]
fn test_validate_skip_tls_verify_and_client_ca_mutually_exclusive() {
let args = WebhookArgs {
@@ -1013,36 +844,6 @@ mod tests {
assert!(args.validate().is_ok());
}
#[test]
fn webhook_tls_warning_redacts_endpoint_details() {
let captured = CapturedLog::default();
let subscriber = tracing_subscriber::fmt()
.with_ansi(false)
.without_time()
.with_max_level(tracing::Level::WARN)
.with_writer(captured.clone())
.finish();
let args = WebhookArgs {
endpoint: Url::parse("https://webhook.test/private?token=secret").expect("webhook endpoint"),
skip_tls_verify: true,
..base_args()
};
tracing::subscriber::with_default(subscriber, || {
WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(
&args,
StaticResolver(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)),
)
.expect("webhook client should build");
});
let logs = captured.contents();
assert!(logs.contains("https://webhook.test"));
for secret in ["/private", "token=secret"] {
assert!(!logs.contains(secret), "TLS warning leaked {secret}: {logs}");
}
}
#[test]
fn test_validate_rejects_loopback_endpoint() {
let args = WebhookArgs {
@@ -1129,20 +930,13 @@ mod tests {
#[tokio::test]
async fn dns_failure_has_stable_health_reason() {
let url = Url::parse("http://unresolvable.test/").expect("invalid test domain URL");
let client = WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(&base_args(), FailingResolver)
.expect("build client");
let url = Url::parse("http://rustfs-health-check.invalid/").expect("invalid test domain URL");
let client = WebhookTarget::<serde_json::Value>::build_http_client(&base_args()).expect("build client");
let health = probe_health_url(&client, &url).await;
assert_eq!(health.state, TargetHealthState::Error);
// On systems with DNS interception (common on macOS), `.invalid` may resolve
// to an interception address, producing `Unreachable` instead of `DnsFailure`.
assert!(
matches!(health.reason, TargetHealthReason::DnsFailure | TargetHealthReason::Unreachable),
"expected DnsFailure or Unreachable, got {:?}",
health.reason
);
assert_eq!(health.reason, TargetHealthReason::DnsFailure);
}
#[tokio::test(start_paused = true)]
@@ -1201,9 +995,7 @@ mod tests {
let _ = tls_stream.read(&mut request);
});
let url = Url::parse(&format!("https://localhost:{}/", address.port())).expect("TLS health URL");
let client =
WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(&base_args(), StaticResolver(address.ip()))
.expect("build client");
let client = WebhookTarget::<serde_json::Value>::build_http_client(&base_args()).expect("build client");
let health = probe_health_url(&client, &url).await;
@@ -1335,14 +1127,12 @@ mod tests {
});
let args = WebhookArgs {
endpoint: Url::parse(&format!("https://localhost:{}/hook", addr.port())).expect("endpoint should parse"),
client_ca: ca_path.to_string_lossy().into_owned(),
..base_args()
};
let client = WebhookTarget::<serde_json::Value>::build_http_client_with_resolver(&args, StaticResolver(addr.ip()))
.expect("build https client");
let client = WebhookTarget::<serde_json::Value>::build_http_client(&args).expect("build https client");
let resp = client
.head(args.endpoint)
.head(format!("https://localhost:{}/hook", addr.port()))
.send()
.await
.expect("https webhook probe should trust configured ca");
+2 -3
View File
@@ -42,7 +42,6 @@ lz4 = { workspace = true, optional = true }
md-5 = { workspace = true, optional = true }
netif = { workspace = true, optional = true }
regex = { workspace = true, optional = true }
reqwest = { workspace = true, optional = true }
rustix = { workspace = true, optional = true, features = ["fs"] }
serde = { workspace = true, optional = true, features = ["derive"] }
sha1 = { workspace = true, optional = true }
@@ -51,7 +50,7 @@ convert_case = { workspace = true, optional = true }
siphasher = { workspace = true, optional = true }
snap = { workspace = true, optional = true }
tempfile = { workspace = true, optional = true }
tokio = { workspace = true, optional = true, features = ["io-util", "net", "time"] }
tokio = { workspace = true, optional = true, features = ["io-util", "time"] }
tracing = { workspace = true }
transform-stream = { workspace = true, optional = true }
url = { workspace = true, optional = true }
@@ -73,7 +72,7 @@ workspace = true
default = ["ip"] # features that are enabled by default
ip = ["dep:local-ip-address"] # ip characteristics and their dependencies
net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:hyper", "dep:tokio"] # network features with DNS resolver
egress = ["ip", "dep:reqwest", "dep:tokio", "dep:url"]
egress = ["ip", "dep:url"]
io = ["dep:tokio"]
path = [] # path manipulation features
compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
+34 -704
View File
@@ -12,24 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashSet;
#[cfg(test)]
use std::collections::{HashMap, VecDeque};
use std::fmt;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::sync::OnceLock;
#[cfg(test)]
use std::sync::{Arc, Mutex};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use url::Url;
/// Comma-separated exact HTTP(S) origins that may resolve to otherwise restricted addresses.
///
/// This is an operator-owned process setting. Target configuration cannot extend it. Link-local,
/// metadata, and unspecified addresses remain forbidden even when their origin appears in the list.
pub const ENV_OUTBOUND_ALLOW_ORIGINS: &str = "RUSTFS_OUTBOUND_ALLOW_ORIGINS";
static OUTBOUND_POLICY: OnceLock<Result<OutboundPolicy, OutboundPolicyError>> = OnceLock::new();
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum OutboundUrlError {
MissingHost,
@@ -49,199 +35,6 @@ impl fmt::Display for OutboundUrlError {
impl std::error::Error for OutboundUrlError {}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum OutboundPolicyError {
NonUnicodeEnvironment,
InvalidAllowedOrigin { position: usize },
MissingHost,
UnsupportedScheme { scheme: String },
UserInfoNotAllowed,
ForbiddenHost { host: String, reason: &'static str },
}
impl fmt::Display for OutboundPolicyError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
OutboundPolicyError::NonUnicodeEnvironment => {
write!(f, "{ENV_OUTBOUND_ALLOW_ORIGINS} must contain valid Unicode")
}
OutboundPolicyError::InvalidAllowedOrigin { position } => {
write!(f, "invalid origin at position {position} in {ENV_OUTBOUND_ALLOW_ORIGINS}")
}
OutboundPolicyError::MissingHost => write!(f, "outbound URL is missing a host"),
OutboundPolicyError::UnsupportedScheme { scheme } => {
write!(f, "outbound URL scheme '{scheme}' is not allowed")
}
OutboundPolicyError::UserInfoNotAllowed => write!(f, "outbound URL userinfo is not allowed"),
OutboundPolicyError::ForbiddenHost { host, reason } => {
write!(f, "outbound URL host '{host}' is not allowed: {reason}")
}
}
}
}
impl std::error::Error for OutboundPolicyError {}
#[derive(Debug, Default, PartialEq, Eq)]
pub struct OutboundPolicy {
allowed_restricted_origins: HashSet<String>,
}
impl OutboundPolicy {
pub fn from_env() -> Result<Self, OutboundPolicyError> {
match std::env::var(ENV_OUTBOUND_ALLOW_ORIGINS) {
Ok(value) => Self::from_allowed_origins(&value),
Err(std::env::VarError::NotPresent) => Ok(Self::default()),
Err(std::env::VarError::NotUnicode(_)) => Err(OutboundPolicyError::NonUnicodeEnvironment),
}
}
pub fn from_env_cached() -> Result<&'static Self, OutboundPolicyError> {
match OUTBOUND_POLICY.get_or_init(Self::from_env) {
Ok(policy) => Ok(policy),
Err(err) => Err(err.clone()),
}
}
pub fn from_allowed_origins(value: &str) -> Result<Self, OutboundPolicyError> {
let mut allowed_restricted_origins = HashSet::new();
if value.trim().is_empty() {
return Ok(Self::default());
}
for (position, entry) in value.split(',').map(str::trim).enumerate() {
let position = position + 1;
if entry.is_empty() {
return Err(OutboundPolicyError::InvalidAllowedOrigin { position });
}
let url = Url::parse(entry).map_err(|_| OutboundPolicyError::InvalidAllowedOrigin { position })?;
let origin = normalized_origin(&url).ok_or(OutboundPolicyError::InvalidAllowedOrigin { position })?;
if url.path() != "/" || url.query().is_some() || url.fragment().is_some() {
return Err(OutboundPolicyError::InvalidAllowedOrigin { position });
}
allowed_restricted_origins.insert(origin);
}
Ok(Self {
allowed_restricted_origins,
})
}
pub fn validate_url(&self, url: &Url) -> Result<(), OutboundPolicyError> {
validate_http_url_shape(url)?;
let raw_host = url.host_str().ok_or(OutboundPolicyError::MissingHost)?;
let normalized_host = raw_host.trim_end_matches('.').trim_matches(['[', ']']);
let result = if normalized_host.eq_ignore_ascii_case("localhost") {
Err("loopback host")
} else if let Ok(ip) = normalized_host.parse::<IpAddr>() {
validate_policy_ip(ip)
} else {
Ok(())
};
match result {
Ok(()) => Ok(()),
Err(reason) if self.allows_restricted_origin(url) && restricted_reason_can_be_overridden(reason) => Ok(()),
Err(reason) => Err(OutboundPolicyError::ForbiddenHost {
host: raw_host.to_string(),
reason,
}),
}
}
pub fn resolver_for(&self, url: &Url) -> Result<OutboundDnsResolver, OutboundPolicyError> {
self.validate_url(url)?;
let host = normalized_host(url).ok_or(OutboundPolicyError::MissingHost)?;
Ok(OutboundDnsResolver::new(host, self.allows_restricted_origin(url)))
}
fn allows_restricted_origin(&self, url: &Url) -> bool {
normalized_origin(url).is_some_and(|origin| self.allowed_restricted_origins.contains(&origin))
}
}
#[derive(Clone, Debug)]
pub struct OutboundDnsResolver {
allowed_restricted_host: Option<String>,
#[cfg(test)]
overrides: Option<DnsOverrideAnswers>,
}
#[cfg(test)]
type DnsOverrideAnswers = Arc<Mutex<HashMap<String, VecDeque<Vec<SocketAddr>>>>>;
impl OutboundDnsResolver {
fn new(host: String, allow_restricted: bool) -> Self {
Self {
allowed_restricted_host: allow_restricted.then_some(host),
#[cfg(test)]
overrides: None,
}
}
#[cfg(test)]
fn with_overrides(mut self, overrides: HashMap<String, Vec<IpAddr>>) -> Self {
let overrides = overrides
.into_iter()
.map(|(host, ips)| {
let addresses = ips.into_iter().map(|ip| SocketAddr::new(ip, 0)).collect();
(host, VecDeque::from([addresses]))
})
.collect();
self.overrides = Some(Arc::new(Mutex::new(overrides)));
self
}
#[cfg(test)]
fn with_override_sequence(mut self, host: &str, answers: Vec<Vec<SocketAddr>>) -> Self {
self.overrides = Some(Arc::new(Mutex::new(HashMap::from([(host.to_string(), VecDeque::from(answers))]))));
self
}
}
impl reqwest::dns::Resolve for OutboundDnsResolver {
fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving {
let host = name.as_str().trim_end_matches('.').to_ascii_lowercase();
let allow_restricted = self.allowed_restricted_host.as_deref() == Some(host.as_str());
#[cfg(test)]
let overrides = self.overrides.clone();
Box::pin(async move {
#[cfg(test)]
let overridden = overrides.as_ref().and_then(|entries| {
let mut entries = entries.lock().expect("test DNS overrides must not be poisoned");
let answers = entries.get_mut(&host)?;
if answers.len() > 1 {
answers.pop_front()
} else {
answers.front().cloned()
}
});
#[cfg(not(test))]
let overridden: Option<Vec<SocketAddr>> = None;
let addresses = if let Some(addresses) = overridden {
addresses
} else {
// Do not use the cached resolver in `crate::net`: every new connection must
// classify the addresses returned at that boundary so DNS rebinding fails closed.
tokio::net::lookup_host((host.as_str(), 0))
.await
.map_err(|err| std::io::Error::new(std::io::ErrorKind::NotFound, err))?
.collect()
};
let addrs = addresses
.into_iter()
.filter(|address| resolved_ip_allowed(address.ip(), allow_restricted))
.collect::<Vec<_>>();
if addrs.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
format!("outbound DNS resolution for '{host}' returned no allowed addresses"),
)
.into());
}
Ok(Box::new(addrs.into_iter()) as reqwest::dns::Addrs)
})
}
}
pub fn validate_outbound_url(url: &Url) -> Result<(), OutboundUrlError> {
let Some(raw_host) = url.host_str() else {
return Err(OutboundUrlError::MissingHost);
@@ -265,168 +58,10 @@ pub fn validate_outbound_url(url: &Url) -> Result<(), OutboundUrlError> {
})
}
fn validate_http_url_shape(url: &Url) -> Result<(), OutboundPolicyError> {
if !matches!(url.scheme(), "http" | "https") {
return Err(OutboundPolicyError::UnsupportedScheme {
scheme: url.scheme().to_string(),
});
}
if !url.username().is_empty() || url.password().is_some() {
return Err(OutboundPolicyError::UserInfoNotAllowed);
}
Ok(())
}
fn normalized_host(url: &Url) -> Option<String> {
url.host_str()
.map(|host| host.trim_end_matches('.').trim_matches(&['[', ']'][..]).to_ascii_lowercase())
}
fn normalized_origin(url: &Url) -> Option<String> {
validate_http_url_shape(url).ok()?;
let host = normalized_host(url)?;
let port = url.port_or_known_default()?;
// `Url::origin()` preserves a trailing DNS dot, but the resolver treats dotted and undotted
// names as the same host. Normalize it here so policy matching follows connection semantics.
if host.parse::<Ipv6Addr>().is_ok() {
Some(format!("{}://[{host}]:{port}", url.scheme()))
} else {
Some(format!("{}://{host}:{port}", url.scheme()))
}
}
fn restricted_reason_can_be_overridden(reason: &str) -> bool {
matches!(
reason,
"loopback host" | "loopback address" | "private address" | "shared address" | "reserved address"
)
}
fn resolved_ip_allowed(ip: IpAddr, allow_restricted: bool) -> bool {
match validate_policy_ip(ip) {
Ok(()) => true,
Err(reason) => allow_restricted && restricted_reason_can_be_overridden(reason),
}
}
fn validate_policy_ip(ip: IpAddr) -> Result<(), &'static str> {
if is_metadata_endpoint(ip) {
return Err("metadata endpoint");
}
let original_v6 = match ip {
IpAddr::V6(v6) => Some(v6),
IpAddr::V4(_) => None,
};
let ip = match original_v6 {
Some(v6) => match policy_embedded_ipv4(v6) {
Some(v4) => IpAddr::V4(v4),
None => IpAddr::V6(v6),
},
None => ip,
};
if is_metadata_endpoint(ip) {
return Err("metadata endpoint");
}
validate_outbound_ip(ip)?;
if original_v6.is_some_and(|v6| v6.segments()[0] == 0x2002) {
return Err("reserved address");
}
match ip {
IpAddr::V4(ipv4) => {
let octets = ipv4.octets();
if octets[0] == 100 && (64..=127).contains(&octets[1]) {
return Err("shared address");
}
if octets[0] == 0
|| octets[0] >= 224
|| (octets[0] == 192 && ((octets[1] == 0 && matches!(octets[2], 0 | 2)) || (octets[1] == 88 && octets[2] == 99)))
|| (octets[0] == 198 && (octets[1] == 18 || octets[1] == 19))
|| (octets[0] == 198 && octets[1] == 51 && octets[2] == 100)
|| (octets[0] == 203 && octets[1] == 0 && octets[2] == 113)
{
return Err("reserved address");
}
}
IpAddr::V6(ipv6) => {
let segments = ipv6.segments();
if segments[0] == 0x0064 && segments[1] == 0xff9b && segments[2] == 1 {
return Err("translation address");
}
if is_special_use_ipv6(ipv6) {
return Err("reserved address");
}
}
}
Ok(())
}
fn is_special_use_ipv6(ip: Ipv6Addr) -> bool {
let segments = ip.segments();
let bits = u128::from_be_bytes(ip.octets());
let ietf_protocol_assignment = segments[0] == 0x2001
&& segments[1] < 0x0200
&& bits != 0x2001_0001_0000_0000_0000_0000_0000_0001
&& bits != 0x2001_0001_0000_0000_0000_0000_0000_0002
&& segments[1] != 0x0003
&& !(segments[1] == 0x0004 && segments[2] == 0x0112)
&& !(0x0020..=0x003f).contains(&segments[1]);
(segments[0] & 0xff00) == 0xff00
|| (segments[0] & 0xffc0) == 0xfec0
|| (segments[0] == 0x0100 && segments[1..4] == [0, 0, 0])
|| ietf_protocol_assignment
|| segments[0] == 0x2002
|| (segments[0] == 0x2001 && segments[1] == 0x0db8)
|| (segments[0] == 0x3fff && segments[1] <= 0x0fff)
|| segments[0] == 0x5f00
}
fn is_metadata_endpoint(ip: IpAddr) -> bool {
match ip {
IpAddr::V4(ipv4) => matches!(
ipv4.octets(),
[169, 254, 169, 254]
| [169, 254, 170, 2]
| [169, 254, 170, 23]
| [169, 254, 0, 23]
| [100, 100, 100, 200]
| [168, 63, 129, 16]
),
IpAddr::V6(ipv6) => matches!(
ipv6.segments(),
[0xfd00, 0x0ec2, 0, 0, 0, 0, 0, 0x0254]
| [0xfd00, 0x0ec2, 0, 0, 0, 0, 0, 0x0023]
| [0xfd20, 0x00ce, 0, 0, 0, 0, 0, 0x0254]
),
}
}
fn policy_embedded_ipv4(v6: Ipv6Addr) -> Option<Ipv4Addr> {
if let Some(v4) = embedded_ipv4(v6) {
return Some(v4);
}
let segments = v6.segments();
if segments[..6] == [0, 0, 0, 0, 0xffff, 0] {
let hi = segments[6].to_be_bytes();
let lo = segments[7].to_be_bytes();
return Some(Ipv4Addr::new(hi[0], hi[1], lo[0], lo[1]));
}
let octets = v6.octets();
if octets[..12] == [0x00, 0x64, 0xff, 0x9b, 0, 0, 0, 0, 0, 0, 0, 0] {
return Some(Ipv4Addr::new(octets[12], octets[13], octets[14], octets[15]));
}
if octets[0..2] == [0x20, 0x02] {
return Some(Ipv4Addr::new(octets[2], octets[3], octets[4], octets[5]));
}
None
}
fn validate_outbound_ip(ip: IpAddr) -> Result<(), &'static str> {
// Reject pure-IPv6 special forms first, before any IPv4 normalization. ::1 (loopback) and ::
// (unspecified) are technically IPv4-compatible forms too, so normalizing first would map
// them to a harmless-looking 0.0.0.1 / 0.0.0.0 and let them through.
if let IpAddr::V6(v6) = ip {
if v6.is_loopback() {
return Err("loopback address");
@@ -442,6 +77,10 @@ fn validate_outbound_ip(ip: IpAddr) -> Result<(), &'static str> {
}
}
// Normalize IPv4-mapped (::ffff:a.b.c.d) AND IPv4-compatible (::a.b.c.d) IPv6 addresses to
// their embedded IPv4 so the IPv4 rules below apply. The std is_* checks on the IPv6 variant
// never inspect the embedded IPv4, so without this an attacker bypasses the guard with e.g.
// ::ffff:127.0.0.1, ::127.0.0.1 (loopback) or ::169.254.169.254 (cloud metadata service).
let ip = match ip {
IpAddr::V6(v6) => match embedded_ipv4(v6) {
Some(v4) => IpAddr::V4(v4),
@@ -453,32 +92,44 @@ fn validate_outbound_ip(ip: IpAddr) -> Result<(), &'static str> {
if ip.is_unspecified() {
return Err("unspecified address");
}
if ip == IpAddr::V4(Ipv4Addr::new(169, 254, 169, 254)) {
return Err("metadata endpoint");
}
if let IpAddr::V4(ipv4) = ip {
if ipv4.is_loopback() {
return Err("loopback address");
}
if ipv4.is_link_local() {
return Err("link-local address");
}
if ipv4.is_private() {
return Err("private address");
match ip {
IpAddr::V4(ipv4) => {
if ipv4.is_loopback() {
return Err("loopback address");
}
if ipv4.is_link_local() {
return Err("link-local address");
}
if ipv4.is_private() {
return Err("private address");
}
}
// Genuine IPv6 (no embedded IPv4) was already classified above.
IpAddr::V6(_) => {}
}
Ok(())
}
/// Extract the embedded IPv4 from an IPv4-mapped (`::ffff:a.b.c.d`) or IPv4-compatible
/// (`::a.b.c.d`) IPv6 address. The pure-IPv6 specials `::` and `::1` are rejected by the caller
/// before this runs, so returning `None` here means a genuine IPv6 host.
fn embedded_ipv4(v6: Ipv6Addr) -> Option<Ipv4Addr> {
if let Some(v4) = v6.to_ipv4_mapped() {
return Some(v4);
}
let segments = v6.segments();
if segments[0..6] == [0, 0, 0, 0, 0, 0] {
let hi = segments[6].to_be_bytes();
let lo = segments[7].to_be_bytes();
// IPv4-compatible: the top 96 bits are zero and the low 32 bits carry the IPv4.
let segs = v6.segments();
if segs[0..6] == [0, 0, 0, 0, 0, 0] {
let hi = segs[6].to_be_bytes();
let lo = segs[7].to_be_bytes();
let v4 = Ipv4Addr::new(hi[0], hi[1], lo[0], lo[1]);
// `::` and `::1` are already handled by the caller; anything else is a real embedded v4.
if !v4.is_unspecified() && v4 != Ipv4Addr::new(0, 0, 0, 1) {
return Some(v4);
}
@@ -488,9 +139,7 @@ fn embedded_ipv4(v6: Ipv6Addr) -> Option<Ipv4Addr> {
#[cfg(test)]
mod tests {
use super::{OutboundPolicy, OutboundUrlError, validate_outbound_url};
use std::collections::HashMap;
use std::net::SocketAddr;
use super::{OutboundUrlError, validate_outbound_url};
use url::Url;
#[test]
@@ -525,17 +174,6 @@ mod tests {
));
}
#[test]
fn validate_outbound_url_rejects_alternate_ipv4_loopback_syntax() {
for endpoint in ["http://2130706433/hook", "http://0177.0.0.1/hook", "http://0x7f000001/hook"] {
let url = Url::parse(endpoint).expect("alternate IPv4 URL should parse");
assert!(
validate_outbound_url(&url).is_err(),
"alternate loopback syntax must be rejected: {endpoint}"
);
}
}
#[test]
fn validate_outbound_url_rejects_private_ip() {
let url = Url::parse("https://10.0.0.5/webhook").expect("private URL should parse");
@@ -549,35 +187,6 @@ mod tests {
));
}
#[test]
fn outbound_policy_rejects_special_use_addresses() {
let policy = OutboundPolicy::default();
for endpoint in [
"http://0.0.0.1/hook",
"http://192.0.2.1/hook",
"http://198.18.0.1/hook",
"http://198.51.100.1/hook",
"http://203.0.113.1/hook",
"http://224.0.0.1/hook",
"http://[100::1]/hook",
"http://[2001:100::1]/hook",
"http://[2001:db8::1]/hook",
"http://[3fff::1]/hook",
"http://[5f00::1]/hook",
"http://[ff02::1]/hook",
] {
let url = Url::parse(endpoint).expect("special-use URL should parse");
assert!(policy.validate_url(&url).is_err(), "special-use address must be rejected: {endpoint}");
}
}
#[test]
fn validate_outbound_url_preserves_legacy_shared_address_behavior() {
let url = Url::parse("http://100.64.0.5/hook").expect("shared URL should parse");
assert!(validate_outbound_url(&url).is_ok());
assert!(OutboundPolicy::default().validate_url(&url).is_err());
}
#[test]
fn validate_outbound_url_rejects_metadata_endpoint() {
let url = Url::parse("http://169.254.169.254/latest/meta-data").expect("metadata URL should parse");
@@ -677,26 +286,6 @@ mod tests {
));
}
#[test]
fn outbound_policy_rejects_embedded_private_destinations() {
let policy = OutboundPolicy::default();
for endpoint in [
"http://[64:ff9b::7f00:1]/hook",
"http://[64:ff9b::a00:1]/hook",
"http://[64:ff9b::a9fe:a9fe]/latest/meta-data",
"http://[2002:7f00:1::]/hook",
"http://[2002:a00:1::]/hook",
"http://[::ffff:0:7f00:1]/hook",
"http://[::ffff:0:a9fe:a9fe]/latest/meta-data",
] {
let url = Url::parse(endpoint).expect("embedded IPv4 URL should parse");
assert!(
policy.validate_url(&url).is_err(),
"embedded private destination must be rejected: {endpoint}"
);
}
}
#[test]
fn validate_outbound_url_rejects_ipv6_loopback_and_unspecified() {
// ::1 / :: must stay rejected even though they look like IPv4-compatible forms.
@@ -717,263 +306,4 @@ mod tests {
}
));
}
#[test]
fn outbound_policy_allows_only_the_exact_operator_origin() {
let policy = OutboundPolicy::from_allowed_origins("http://127.0.0.1:9443").expect("operator origin should parse");
assert!(
policy
.validate_url(&Url::parse("http://127.0.0.1:9443/hook").expect("allowed URL"))
.is_ok()
);
assert!(
policy
.validate_url(&Url::parse("http://127.0.0.1:9444/hook").expect("wrong-port URL"))
.is_err(),
"an allowlisted origin must not authorize another port"
);
assert!(
policy
.validate_url(&Url::parse("http://[::1]:9443/hook").expect("different-host URL"))
.is_err(),
"an allowlisted origin must not authorize a different host"
);
let shared_policy =
OutboundPolicy::from_allowed_origins("http://100.64.0.5:9443").expect("shared operator origin should parse");
assert!(
shared_policy
.validate_url(&Url::parse("http://100.64.0.5:9443/hook").expect("shared URL"))
.is_ok(),
"an exact operator origin may opt into an internal shared address"
);
}
#[test]
fn outbound_policy_never_allows_metadata_or_unspecified_addresses() {
let policy = OutboundPolicy::from_allowed_origins(
"http://169.254.1.10,http://169.254.169.254,http://169.254.170.2,http://169.254.170.23,http://169.254.0.23,http://100.100.100.200,http://168.63.129.16,http://[fd00:ec2::254],http://[fd00:ec2::23],http://[fd20:ce::254],http://[64:ff9b:1::1],http://0.0.0.0",
)
.expect("syntactically valid origins should parse");
for endpoint in [
"http://169.254.1.10/hook",
"http://169.254.169.254/latest",
"http://169.254.170.2/credentials",
"http://169.254.170.23/v1/credentials",
"http://169.254.0.23/latest/meta-data",
"http://100.100.100.200/latest/meta-data",
"http://168.63.129.16/machine?comp=goalstate",
"http://[fd00:ec2::254]/latest",
"http://[fd00:ec2::23]/v1/credentials",
"http://[fd20:ce::254]/computeMetadata/v1",
"http://[64:ff9b:1::1]/hook",
"http://0.0.0.0/hook",
] {
assert!(
policy
.validate_url(&Url::parse(endpoint).expect("test endpoint should parse"))
.is_err(),
"endpoint must remain forbidden: {endpoint}"
);
}
}
#[test]
fn outbound_policy_rejects_non_http_and_userinfo_origins() {
for origin in [
"ftp://webhook.internal",
"https://user@webhook.internal",
"https://webhook.internal/path",
] {
assert!(
OutboundPolicy::from_allowed_origins(origin).is_err(),
"invalid operator origin must fail closed: {origin}"
);
}
assert!(OutboundPolicy::from_allowed_origins("https://one.example,,https://two.example").is_err());
}
#[test]
fn outbound_policy_rejects_non_http_and_userinfo_targets() {
let policy = OutboundPolicy::default();
for endpoint in ["ftp://webhook.test/hook", "https://user:secret@webhook.test/hook"] {
assert!(
policy
.validate_url(&Url::parse(endpoint).expect("target URL should parse"))
.is_err(),
"target must be rejected: {endpoint}"
);
}
}
#[tokio::test]
async fn outbound_dns_resolver_filters_rebinding_and_mixed_answers() {
let policy = OutboundPolicy::default();
let endpoint = Url::parse("https://webhook.test/hook").expect("endpoint should parse");
let resolver = policy
.resolver_for(&endpoint)
.expect("public hostname should be accepted")
.with_overrides(HashMap::from([
(
"webhook.test".to_string(),
vec![
"10.0.0.5".parse().expect("private IP"),
"100.64.0.5".parse().expect("shared IP"),
"100.100.100.200".parse().expect("metadata IP"),
"8.8.8.8".parse().expect("public IP"),
"::ffff:127.0.0.1".parse().expect("mapped loopback IP"),
],
),
("rebound.test".to_string(), vec!["127.0.0.1".parse().expect("loopback IP")]),
]));
let addrs = reqwest::dns::Resolve::resolve(&resolver, "webhook.test".parse().expect("resolver hostname"))
.await
.expect("one public address should remain")
.map(|addr| addr.ip())
.collect::<Vec<_>>();
assert_eq!(addrs, vec!["8.8.8.8".parse::<std::net::IpAddr>().expect("public IP")]);
assert!(
reqwest::dns::Resolve::resolve(&resolver, "rebound.test".parse().expect("resolver hostname"))
.await
.is_err(),
"a rebound answer containing only restricted addresses must fail closed"
);
}
#[tokio::test]
async fn exact_operator_origin_allows_private_dns_answers_for_that_host_only() {
let policy = OutboundPolicy::from_allowed_origins("https://webhook.test:9443").expect("operator origin should parse");
let resolver = policy
.resolver_for(&Url::parse("https://webhook.test:9443/hook").expect("endpoint should parse"))
.expect("allowlisted endpoint should be accepted")
.with_overrides(HashMap::from([
("webhook.test".to_string(), vec!["10.0.0.5".parse().expect("private IP")]),
("other.test".to_string(), vec!["10.0.0.6".parse().expect("private IP")]),
]));
let allowed = reqwest::dns::Resolve::resolve(&resolver, "webhook.test".parse().expect("resolver hostname"))
.await
.expect("allowlisted host should resolve")
.count();
assert_eq!(allowed, 1);
assert!(
reqwest::dns::Resolve::resolve(&resolver, "other.test".parse().expect("resolver hostname"))
.await
.is_err(),
"the allowlist must not authorize another private hostname"
);
}
#[tokio::test]
async fn exact_operator_origin_never_allows_metadata_dns_answers() {
let policy = OutboundPolicy::from_allowed_origins("https://webhook.test:9443").expect("operator origin should parse");
let resolver = policy
.resolver_for(&Url::parse("https://webhook.test:9443/hook").expect("endpoint should parse"))
.expect("allowlisted endpoint should be accepted")
.with_overrides(HashMap::from([(
"webhook.test".to_string(),
vec![
"169.254.170.23".parse().expect("EKS credential IP"),
"169.254.0.23".parse().expect("Tencent metadata IP"),
"fd00:ec2::23".parse().expect("EKS credential IPv6"),
"fd20:ce::254".parse().expect("GCP metadata IPv6"),
"0.0.0.0".parse().expect("unspecified IP"),
],
)]));
assert!(
reqwest::dns::Resolve::resolve(&resolver, "webhook.test".parse().expect("resolver hostname"))
.await
.is_err(),
"metadata and unspecified answers must remain forbidden for an allowlisted origin"
);
}
#[tokio::test]
async fn outbound_dns_resolver_preserves_ipv6_scope_id() {
use std::net::SocketAddrV6;
let policy = OutboundPolicy::default();
let scoped = SocketAddr::V6(SocketAddrV6::new("2001:4860:4860::8888".parse().expect("public IPv6"), 0, 0, 7));
let resolver = policy
.resolver_for(&Url::parse("https://webhook.test:9443/hook").expect("endpoint should parse"))
.expect("public endpoint should be accepted")
.with_override_sequence("webhook.test", vec![vec![scoped]]);
let resolved = reqwest::dns::Resolve::resolve(&resolver, "webhook.test".parse().expect("resolver hostname"))
.await
.expect("public scoped address should resolve")
.collect::<Vec<_>>();
assert_eq!(resolved, vec![scoped]);
}
#[tokio::test]
async fn reqwest_rechecks_dns_policy_for_each_new_connection() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind first connection listener");
let address = listener.local_addr().expect("first connection address");
let endpoint = Url::parse(&format!("http://rebind.test:{}/hook", address.port())).expect("endpoint should parse");
let policy = OutboundPolicy::from_allowed_origins(&endpoint.origin().ascii_serialization())
.expect("loopback test origin should be explicitly allowed");
let resolver = policy
.resolver_for(&endpoint)
.expect("allowlisted endpoint should be accepted")
.with_override_sequence(
"rebind.test",
vec![
vec![SocketAddr::new(address.ip(), 0)],
vec![SocketAddr::new("169.254.169.254".parse().expect("metadata IP"), 0)],
],
);
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("accept first connection");
let mut request = [0_u8; 1024];
let _ = stream.read(&mut request).await.expect("read first request");
stream
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.await
.expect("write first response");
});
let client = reqwest::Client::builder()
.no_proxy()
.dns_resolver(resolver)
.timeout(std::time::Duration::from_secs(2))
.build()
.expect("build test client");
assert_eq!(
client
.get(endpoint.clone())
.send()
.await
.expect("first request should connect")
.status(),
reqwest::StatusCode::NO_CONTENT
);
server.await.expect("first connection server should finish");
let error = client
.get(endpoint)
.send()
.await
.expect_err("the second connection must reject a metadata DNS answer");
let mut error_chain = vec![error.to_string()];
let mut source = std::error::Error::source(&error);
while let Some(error) = source {
error_chain.push(error.to_string());
source = error.source();
}
assert!(
error_chain
.iter()
.any(|message| message.contains("returned no allowed addresses")),
"request must fail in the DNS policy layer: {error_chain:?}"
);
}
}
+56
View File
@@ -20,12 +20,26 @@
use http::{HeaderMap, HeaderValue};
use std::borrow::Cow;
use std::collections::HashMap;
use std::error::Error;
use std::fmt::{Display, Formatter};
const RUSTFS_PREFIX: &str = "x-rustfs-";
const MINIO_PREFIX: &str = "x-minio-";
const MINIO_ENCRYPTION_PREFIX: &str = "x-minio-encryption-";
const RUSTFS_ENCRYPTION_PREFIX: &str = "x-rustfs-encryption-";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ConflictingMetadataValue;
impl Display for ConflictingMetadataValue {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
f.write_str("metadata contains conflicting values for the same case-insensitive key")
}
}
impl Error for ConflictingMetadataValue {}
// Suffix constants (part after x-rustfs- or x-minio-). Use with get_header/insert_header.
pub const SUFFIX_FORCE_DELETE: &str = "force-delete";
pub const SUFFIX_INCLUDE_DELETED: &str = "include-deleted";
@@ -84,6 +98,24 @@ pub fn get_header_map(map: &std::collections::HashMap<String, String>, suffix: &
map.get(&rk).cloned().or_else(|| map.get(&mk).cloned())
}
/// Returns the value when every case-insensitive occurrence of `name` agrees.
pub fn get_consistent_metadata_value<'a>(
metadata: &'a HashMap<String, String>,
name: &str,
) -> Result<Option<&'a str>, ConflictingMetadataValue> {
let mut value = None;
for (candidate, candidate_value) in metadata {
if !candidate.eq_ignore_ascii_case(name) {
continue;
}
if value.is_some_and(|existing| existing != candidate_value) {
return Err(ConflictingMetadataValue);
}
value = Some(candidate_value.as_str());
}
Ok(value)
}
/// Insert into HashMap with both x-rustfs-{suffix} and x-minio-{suffix}.
pub fn insert_header_map(map: &mut std::collections::HashMap<String, String>, suffix: &str, value: impl Into<String>) {
let v = value.into();
@@ -120,4 +152,28 @@ mod tests {
headers2.insert("X-Rustfs-Force-Delete", HeaderValue::from_static("true"));
assert_eq!(get_header(&headers2, SUFFIX_FORCE_DELETE).as_deref(), Some("true"));
}
#[test]
fn test_get_consistent_metadata_value() {
let mut metadata = HashMap::new();
assert_eq!(
get_consistent_metadata_value(&metadata, "x-test").expect("missing value should be valid"),
None
);
metadata.insert("X-Test".to_string(), "value".to_string());
assert_eq!(
get_consistent_metadata_value(&metadata, "x-test").expect("single value should be valid"),
Some("value")
);
metadata.insert("x-test".to_string(), "value".to_string());
assert_eq!(
get_consistent_metadata_value(&metadata, "x-test").expect("matching values should be valid"),
Some("value")
);
metadata.insert("x-TEST".to_string(), "other".to_string());
assert!(get_consistent_metadata_value(&metadata, "x-test").is_err());
}
}
+1
View File
@@ -30,6 +30,7 @@ Two rules keep this directory healthy:
## Contracts & invariants
- [erasure-coding.md](erasure-coding.md) — normative erasure-coding algorithm and on-disk (`xl.meta`) compatibility contract; the frozen invariants for all user-data read/write, encode/decode, quorum, heal, and decode tolerance
- [minio-sse-key-hierarchy.md](minio-sse-key-hierarchy.md) — MinIO-compatible SSE key hierarchy, wrapping algorithms, DARE 2.0 stream format, metadata persistence, and RustFS layer boundaries
- [placement-repair-invariants.md](placement-repair-invariants.md)
- [unified-object-generation.md](unified-object-generation.md) — single per-object generation authority (fencing epoch, transport/encoding/proto/mixed-version contracts)
- [runtime-capability-contracts.md](runtime-capability-contracts.md)
@@ -12,6 +12,7 @@ for later deletion.
## Open Items
- `rustfs-5063` legacy SSE-S3 KMS envelopes: releases before provider routing was separated could wrap an AES256 object's DEK with KMS, so readers identify the persisted KMS envelope and retain KMS decryption for those objects. Remove this compatibility path after the SSE-S3 migration has rewrapped every referenced legacy DEK with the local SSE-S3 key provider.
- `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability.
- `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request.
- `heal-status-rpc-v1` node heal status capability: new peers treat an unimplemented BackgroundHealStatus RPC as an explicitly incomplete rolling-upgrade response. Remove the fallback after the minimum supported RustFS peer version implements BackgroundHealStatus.
@@ -0,0 +1,983 @@
# MinIO SSE Key Hierarchy & RustFS Compatibility Contract
This document is the core reference for RustFS implementation and review of
MinIO-compatible Server-Side Encryption (SSE). It defines key terminology,
derivation relationships, inter-layer responsibilities, persistence formats,
data encryption formats for single-part and multipart objects, and the
compatibility behavior required when reading MinIO objects.
This document describes a long-term architecture contract, not a migration
plan for a specific implementation phase. The historical RustFS direct-stream
key format belongs to the read-compatibility surface and does not alter the
MinIO compatibility target specified herein.
## 1. Basis and Scope
This document takes the following source snapshots as its factual baseline:
- MinIO commit
[`7aac2a2c5b7c882e68c1ce017d8256be2feea27f`](https://github.com/minio/minio/tree/7aac2a2c5b7c882e68c1ce017d8256be2feea27f):
`cmd/encryption-v1.go`, `internal/crypto/key.go`,
`internal/crypto/metadata.go`, `internal/crypto/sse-*.go`.
- RustFS commit `5ea9a1fd8f3c1b39154b27fd419cde76abc68f3b`:
[SSE boundary](../../rustfs/src/storage/sse.rs),
[RIO v2 encryption stream](../../crates/rio-v2/src/encrypt_reader.rs), and
[MinIO generated fixtures](../../crates/rio-v2/tests/minio_generated_fixtures.rs).
- MinIO `sio v0.4.1` DARE 2.0 package format; the RustFS compatible
implementation is fixed in
[RIO v2 encryption stream](../../crates/rio-v2/src/encrypt_reader.rs).
This document covers:
- SSE-C, SSE-S3, and SSE-KMS;
- KMS envelope encryption, object key derivation, and wrapping;
- The exact meanings of `ObjectKey`, `sealingKey`, and `SealedKey`;
- The SIO/RIO DARE 2.0 data stream format;
- Multipart part keys;
- MinIO internal object metadata;
- RustFS write, read, and historical compatibility boundaries.
This document does not specify how a KMS service internally protects its KMS
master key, nor does it replace the S3 public request header and error code
API compatibility specification.
## 2. Normative Language
The terms "MUST", "MUST NOT", "SHOULD", and "MAY" in this document carry
their RFC 2119 meanings. If an implementation conflicts with this document,
the behavior that can read real MinIO objects without degrading the security
boundary takes precedence, and this document SHALL be revised accordingly.
## 3. Terminology: Named by Cryptographic Role
`key` and `data key` often refer to different roles in KMS APIs versus the
object encryption layer. Implementations and reviews MUST use the role names
in the table below as their primary reference and MUST NOT infer purpose from
variable names alone.
| Canonical Term | MinIO / RustFS Source Name | Length | Lifetime | Cryptographic Role |
|---|---|---|---|---|
| KMS master key | KMS key / master key | KMS-defined | KMS-internal only | Root key; protects the KMS data key |
| KMS data key plaintext | MinIO `key.Plaintext`; RustFS `DataKey.plaintext_key` | 32 B | Transient, in-process only | Parent/external key of the object envelope; NOT the `ObjectKey` that directly encrypts object content |
| KMS data key ciphertext | MinIO `key.Ciphertext`; RustFS `encrypted_data_key` | KMS-defined | Persisted | KMS data key plaintext wrapped under the KMS master key |
| Local SSE-S3 master key | RustFS local SSE-S3 provider master key | 32 B | Process-lifetime secret | Parent/external key for local/K/V SSE-S3; MUST NOT be persisted in object metadata |
| SSE-C customer key | SSE-C key from the client request | 32 B | In-memory for the request lifetime | Parent/external key for SSE-C |
| object-key generation nonce | Random value inside `crypto.GenerateKey` | 32 B | Discarded after derivation | Ensures distinct `ObjectKey` values under the same parent key |
| **content-encryption DEK / CEK** | **MinIO `ObjectKey`; RustFS object-key mode `key_bytes`** | **32 B** | **In-memory only** | **The key that actually encrypts object content; derives part keys for multipart** |
| key-wrapping IV | MinIO `SealedKey.IV` / `MetaIV`; RustFS `ManagedSealedKey.iv` | 32 B | Persisted | Contributes to deriving the per-object KEK |
| **per-object KEK** | **MinIO local variable `sealingKey`; RustFS `sealing_key`** | **32 B** | **Derived transiently, never persisted** | **Wraps and unwraps `ObjectKey`** |
| wrapped ObjectKey / wrapped content DEK | MinIO `SealedKey.Key`; Base64-decoded value of the corresponding metadata field | 64 B | Persisted | DARE package of `ObjectKey` AEAD-encrypted under the per-object KEK; new writes are DARE 2.0, legacy SSE-C may be DARE 1.0 |
| `SealedKey` logical record | MinIO `SealedKey { Key, IV, Algorithm }`; RustFS `ManagedSealedKey` plus algorithm constant | 64 B + 32 B + algorithm name | Persisted as multiple metadata fields | The complete ObjectKey wrapping record; NOT an object content encryption key |
| DARE stream nonce | Nonce material in the SIO/RIO DARE header | 12 B | Carried in every package header | Combined with the package sequence number to form the AEAD nonce; independent of the key-wrapping IV |
| multipart part key | MinIO `DerivePartKey`; RustFS `derive_part_key` | 32 B | In-memory only | Content-encryption DEK for each part |
### 3.1 Ambiguities That MUST Be Avoided
1. `SealedKey` is NOT "the key that participates in object content
encryption"; it is the logical record of a wrapped `ObjectKey` and its
unwrapping parameters.
2. `sealingKey` IS the per-object KEK; it is not an additional key layer
beyond the KEK.
3. KMS calls its generation output a data key, but in MinIO's object
encryption layer that plaintext data key serves as the parent/external
key; the key that actually encrypts object content is the subsequently
derived `ObjectKey`.
4. `SealedKey.IV` is a 32 B key-wrapping IV; the DARE data stream nonce is
12 B. The two MUST NOT be reused, stored interchangeably, or derived from
each other.
## 4. Overall Key Hierarchy
```mermaid
flowchart TB
subgraph ROOT["Upstream Key Sources"]
KMSMK["KMS master key<br/>root KEK; KMS-internal only"]
KMSGEN["KMS GenerateKey<br/>carrying KMS AssociatedData"]
KMSPT["KMS data key plaintext<br/>32 B; parent/external key; in-memory only"]
KMSCT["KMS data key ciphertext<br/>KMS-wrapped; persisted"]
SSES3LOCAL["Local SSE-S3 master key<br/>32 B; server-managed parent key"]
SSECK["SSE-C customer key<br/>32 B; parent/external key; MUST NOT be persisted"]
KMSMK --> KMSGEN
KMSGEN --> KMSPT
KMSGEN --> KMSCT
end
subgraph SSE["SSE Layer: Key Hierarchy and Metadata Semantics"]
PARENT["parent/external key"]
OGN["object-key generation nonce<br/>random 32 B; discarded after derivation"]
OKEY["ObjectKey<br/>content-encryption DEK / CEK<br/>32 B; in-memory only"]
WIV["key-wrapping IV<br/>random 32 B; persisted"]
BIND["domain + algorithm + bucket/object"]
KEK["sealingKey<br/>per-object KEK<br/>32 B; never persisted"]
SEALED["SealedKey.Key<br/>wrapped ObjectKey<br/>64 B DARE 2.0 package; persisted"]
PARENT -->|"HMAC-SHA256"| OKEY
OGN -->|"context ∥ nonce"| OKEY
PARENT -->|"HMAC-SHA256"| KEK
WIV --> KEK
BIND --> KEK
KEK -->|"AEAD key"| SEALED
OKEY -->|"32 B plaintext"| SEALED
end
subgraph RIO["SIO / RIO Layer: Object Data Stream"]
PLAIN["Object plaintext"]
PART["multipart only<br/>partKey = HMAC-SHA256(ObjectKey, LE32(partNumber))"]
DARE["DARE 2.0 AEAD packages<br/>AES-256-GCM / ChaCha20-Poly1305"]
CIPHER["Object ciphertext stream"]
OKEY -->|"single-part key"| DARE
OKEY --> PART
PART -->|"multipart part key"| DARE
PLAIN --> DARE
DARE --> CIPHER
end
KMSPT --> PARENT
SSES3LOCAL --> PARENT
SSECK --> PARENT
subgraph STORE["Persistence Boundary"]
META["Object SSE metadata<br/>wrapped ObjectKey + wrapping IV + algorithm<br/>+ KMS key ID/ciphertext/context"]
BODY["Object data region<br/>DARE 2.0 ciphertext stream"]
FORBID["Forbidden to persist or log<br/>KMS data key plaintext<br/>SSE-C customer key<br/>ObjectKey / part key<br/>sealingKey"]
end
KMSCT --> META
WIV --> META
SEALED --> META
CIPHER --> BODY
```
The minimal equivalence relationships are:
```text
ObjectKey = content-encryption DEK / CEK
sealingKey = per-object KEK
SealedKey.Key = Wrap(sealingKey, ObjectKey) = wrapped content DEK
```
## 5. Precise Key Transformations
The concatenation operator `||` below denotes byte-string concatenation, and
integers use little-endian encoding.
### 5.1 Origin of the Parent/External Key
| SSE Mode / Provider | Parent/External Key | Upstream Action |
|---|---|---|
| SSE-C | 32 B customer key, client-provided and verified | No KMS call; customer key MUST NOT be persisted |
| SSE-S3 local/K/V | 32 B local SSE-S3 master key | No KMS call; the local parent key seals the per-object `ObjectKey` |
| SSE-S3 KMS compatibility | KMS data key plaintext generated by KMS/provider | Calls `GenerateKey` with the default service-side key ID and object context |
| SSE-KMS | KMS data key plaintext generated by KMS | Calls `GenerateKey` with the requested or default KMS key ID and KMS encryption context |
Provider choice and object write format are independent. RustFS SSE-S3 writes
MUST use the local SSE-S3 provider even when the MinIO ObjectKey write format
is enabled; configuring KMS MUST NOT change SSE-S3 write routing. SSE-KMS
writes MUST use KMS and MUST NOT fall back to the local SSE-S3 provider.
KMS-backed SSE-S3 compatibility objects and SSE-KMS objects MUST preserve
both the KMS key ID and KMS data key ciphertext so that the read path can
recover the same parent/external key via KMS. Local/K/V SSE-S3 ObjectKey
objects omit both fields simultaneously and seal `ObjectKey` directly under
the local SSE-S3 parent key. Having only one KMS field present, or either
present field empty, is corrupt metadata and MUST be rejected.
KMS AssociatedData MUST exactly reproduce MinIO:
```text
objectPath = path.Join(bucket, object)
SSE-S3 KMS compatibility:
kmsContext = { bucket: objectPath }
SSE-KMS:
storedContext = exact copy of the client-provided context
kmsContext = copy(storedContext)
if bucket is absent from kmsContext:
kmsContext[bucket] = objectPath
```
For SSE-KMS, if the client has explicitly provided an entry whose key equals
the current bucket name, MinIO preserves that value and does not overwrite
it; reads MUST reconstruct the same rule using the persisted `storedContext`.
The augmented `kmsContext` used for the KMS call MUST NOT overwrite the
client context that SHALL be persisted as-is. Copy, rename, and rewrap MUST
reproduce the same context rules for the corresponding source/target
operations and MUST NOT transparently forward a temporary map that lacks the
required object binding.
### 5.2 ObjectKey: Object Content DEK
MinIO's computation is:
```text
objectKeyGenerationNonce = CSPRNG(32)
ObjectKey = HMAC-SHA256(
key = parentExternalKey,
data = UTF8("object-encryption-key generation")
|| objectKeyGenerationNonce
)
```
Constraints:
- `ObjectKey` MUST be 32 B.
- `objectKeyGenerationNonce` is used for exactly one derivation and is not
persisted.
- `ObjectKey` does not include bucket/object; path binding occurs in the
per-object KEK derivation.
- `ObjectKey` MUST NOT appear in object metadata, logs, error text, or debug
output.
### 5.3 sealingKey: Per-Object KEK
```text
keyWrappingIV = CSPRNG(32)
sealingKey = HMAC-SHA256(
key = parentExternalKey,
data = keyWrappingIV
|| UTF8(domain)
|| UTF8("DAREv2-HMAC-SHA256")
|| UTF8(canonicalBucketObjectPath)
)
```
`domain` MUST exactly match the SSE mode:
| SSE Mode | domain |
|---|---|
| SSE-C | `SSE-C` |
| SSE-S3 | `SSE-S3` |
| SSE-KMS | `SSE-KMS` |
`canonicalBucketObjectPath` MUST be consistent with MinIO's
`path.Join(bucket, object)` semantics. Any change to the path, domain,
algorithm identifier, or IV will cause unwrapping authentication failure.
Therefore, copying or renaming an object cannot simply carry over the old
`SealedKey`; it MUST re-seal against the target path, or perform a
semantically equivalent key rotation.
This canonicalization is part of the MinIO sealed-key format and SHALL only
be reproduced exactly within the sealing compatibility boundary; general
path-cleaning functions that resolve `.` or `..` MUST NOT be applied to
ordinary S3 object key paths based on this.
For KMS-backed SSE-S3 compatibility objects and SSE-KMS, a target path change
may also change the KMS AssociatedData. In that case, merely re-sealing
`ObjectKey` while keeping the old KMS data key ciphertext is insufficient;
the parent data key MUST be generated or rewrapped under the target effective
KMS context, and that parent key MUST then be used to produce a new
`SealedKey` for the target path. Only when the client's explicit SSE-KMS
context makes the source and target effective AssociatedData identical may
the corresponding KMS envelope be reused after KMS semantics have been
verified. MinIO's normal key rotation path generates a new KMS data key and
then re-seals `ObjectKey`.
### 5.4 SealedKey: Wrapped ObjectKey
```text
SealedKey.Key = DAREv2-AEAD-Encrypt(
key = sealingKey,
plaintext = ObjectKey,
aad = DARE header[0:4],
nonce = DARE package nonce
)
```
The 32 B `ObjectKey` is encoded as a single DARE 2.0 package:
```text
16 B DARE header
+ 32 B ciphertext
+ 16 B AEAD tag
= 64 B SealedKey.Key
```
The MinIO writer may write AES-256-GCM or ChaCha20-Poly1305 depending on
`sio` cipher-suite selection. Compatible readers MUST accept both valid
cipher IDs:
| Cipher Suite | DARE Cipher ID |
|---|---:|
| AES-256-GCM | `0x00` |
| ChaCha20-Poly1305 | `0x01` |
The complete logical record also includes:
```text
SealedKey {
Key: [u8; 64], // wrapped ObjectKey package
IV: [u8; 32], // keyWrappingIV
Algorithm: "DAREv2-HMAC-SHA256"
}
```
The three components are persisted separately in object metadata, not
serialized as a single structure.
### 5.5 Legacy MinIO SSE-C Seal
The MinIO reader also accepts the historical SSE-C algorithm `DARE-SHA256`:
```text
legacySealingKey = SHA256(parentExternalKey || keyWrappingIV)
ObjectKey = sio.Decrypt(
key = legacySealingKey,
minVersion = DARE 1.0,
input = SealedKey.Key
)
```
This format has no domain or bucket/object binding and is permitted only as
a read-only compatibility path for historical objects:
- MinIO sets `MinVersion = DARE 1.0`, so this algorithm branch may read
DARE 1.0 or DARE 2.0 packages produced with the legacy key derivation; it
MUST NOT be implemented as "header must be v1";
- New writes, copy targets, and rewraps MUST NOT produce `DARE-SHA256`;
- SSE-S3 and SSE-KMS MUST NOT accept this algorithm;
- Legacy identification MUST be conditioned on the SSE-C metadata shape and
MUST NOT become a downgrade channel for arbitrarily corrupted v2 metadata;
- After a successful read, if a rewrite occurs, the object SHOULD be
upgraded to the current `DAREv2-HMAC-SHA256` format.
## 6. DARE 2.0 Format for Object Data
The SSE layer hands the plaintext `ObjectKey` to SIO/RIO. SIO/RIO is not
responsible for:
- Calling KMS;
- Parsing SSE-C request headers;
- Generating or unwrapping `SealedKey`;
- Interpreting SSE-S3, SSE-KMS, bucket/object, or KMS context;
- Deciding SSE metadata fields.
SIO/RIO is only responsible for encrypting and decrypting the DARE data
stream using the supplied content DEK.
### 6.1 Physical Format of Each Data Package
```text
DARE 2.0 package
┌──────────────────┬──────────────────────┬──────────────────┐
│ 16-byte header │ 1..65536 B ciphertext│ 16-byte AEAD tag │
└──────────────────┴──────────────────────┴──────────────────┘
```
Header:
| Byte | Field | Meaning |
|---:|---|---|
| `0` | version | `0x20`, DARE 2.0 |
| `1` | cipher suite | `0x00` AES-256-GCM; `0x01` ChaCha20-Poly1305 |
| `23` (`[2:4]`) | payload length minus one | `LE16(plaintextLength - 1)` |
| `415` (`[4:16]`) | stream nonce material | 12 B; the high bit also carries the final-package flag |
A zero-length plaintext is a special valid case: its ciphertext stream is
also zero bytes and produces no DARE packages, hence no final-package flag.
Only after at least one package has been observed and EOF is reached MUST
the last package be required to carry the final flag; an empty stream MUST
NOT be misclassified as truncated.
For package sequence number `sequenceNumber`:
```text
packageNonce = header[4:16]
packageNonce[8:12] ^= LE32(sequenceNumber)
ciphertext || tag = AEAD-Encrypt(
key = ObjectKey or partKey,
nonce = packageNonce,
plaintext = packagePlaintext,
aad = header[0:4]
)
```
Constraints:
- The first package of a given DARE stream has sequence number `0`,
monotonically increasing thereafter.
- The final-package flag MUST be covered by AEAD authentication.
- The reader MUST verify the complete AEAD tag before releasing that
package's plaintext to the upper layer.
- The header nonce is the DARE stream nonce, NOT `SealedKey.IV`.
- Maximum plaintext per package is 64 KiB; each package adds a fixed 32 B
framing overhead.
### 6.2 Multipart Part Key
Multipart objects do not directly use `ObjectKey` to encrypt all parts.
Each part first derives:
```text
partKey = HMAC-SHA256(
key = ObjectKey,
data = LE32(partNumber)
)
```
Each part is then an independent DARE stream:
- Uses the corresponding `partKey`;
- The MinIO writer uses
`SHA256(fmt.Append(nil, uploadID, partNumber))[0:12]` as the stream
nonce; here `fmt.Append` is Go textual concatenation of upload ID and part
number;
- Sequence number restarts from `0`;
- Part boundaries MUST be accurately recovered from the actual/encrypted
part sizes in metadata.
Decimal strings, network byte order, or zero-based part indices MUST NOT
replace `LE32(partNumber)`.
The part number in the partKey derivation is `LE32`, but the part number in
the MinIO stream nonce input is the Go textual representation; the two
encodings differ. The DARE header carries its own nonce, so readers verify
against the header for interoperability. MinIO's deterministic nonce is a
recognized historical write behavior, not a requirement for new RustFS
writes: re-uploading different plaintext under the same upload ID and part
number would reuse the same AEAD key/nonce with that same partKey. New
RustFS writes MUST generate a random, non-repeating stream nonce for each
part write; this is DARE-semantically interoperable with the MinIO reader
but does not target byte-level identical output with the MinIO writer.
### 6.3 Derivation Keys for ETag and Other Internal Object Metadata
In addition to encrypting object content, `ObjectKey` also serves as the
root for several internal metadata sub-keys. This logic belongs at the
SSE/object-metadata boundary, not in RIO's `SealedKey` management.
MinIO's derivation for a non-empty ETag:
```text
etagKey = HMAC-SHA256(ObjectKey, UTF8("SSE-etag"))
sealedETag = DARE-Encrypt(etagKey, etag)
```
A typical 16 B MD5 ETag becomes a `16 B header + 16 B ciphertext + 16 B tag
= 48 B` sealed ETag. An empty ETag remains empty; on read, a 16 B backend
ETag is treated as an unsealed plain ETag and no decryption is performed.
The ETag visible to S3 clients MUST be correctly unsealed/formatted and MUST
NOT directly expose backend sealed bytes.
MinIO's general-purpose internal metadata encrypter uses:
```text
metadataKey = HMAC-SHA256(ObjectKey, UTF8(baseKey))
sealedValue = DARE-Encrypt(metadataKey, value)
```
For example, the compression index uses an independent `baseKey`. Each
`baseKey` is a key-domain label and MUST exactly match MinIO; empty values
remain empty. Implementations MUST NOT directly reuse `ObjectKey` with the
same AEAD nonce to encrypt these metadata values.
## 7. MinIO Internal Metadata Persistence Contract
These fields are internal object metadata; not all of them should be
returned to S3 clients. Binary values are standard Base64 encoded.
RustFS writes of internal object metadata MUST additionally observe the
repository's twin-key rule: write the same semantic value under both
`x-rustfs-internal-<suffix>` and `x-minio-internal-<suffix>`, and on read
prefer the RustFS key while remaining compatible with objects that have only
the MinIO key. The table below lists the MinIO keys required for MinIO
interoperability; the RustFS twin key does not replace them. Write, delete,
and rotate operations MUST handle both keys simultaneously to prevent stale
values from "resurrecting" on the next read.
### 7.1 Common Fields
| MinIO Metadata Key | Value | Required When |
|---|---|---|
| `X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm` | `DAREv2-HMAC-SHA256` | MinIO-style sealed ObjectKey |
| `X-Minio-Internal-Server-Side-Encryption-Iv` | Base64(32 B key-wrapping IV) | MinIO-style sealed ObjectKey |
| `X-Minio-Internal-Encrypted-Multipart` | multipart marker | Encrypted multipart object |
### 7.2 Per-SSE-Mode Fields
| Mode | Wrapped ObjectKey Field | KMS Key ID | KMS Data Key Ciphertext | KMS Context |
|---|---|---|---|---|
| SSE-C | `X-Minio-Internal-Server-Side-Encryption-Sealed-Key` | Not stored | Not stored | Not stored |
| SSE-S3 local/K/V | `X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key` | Not stored | Not stored | Not stored |
| SSE-S3 KMS compatibility | `X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key` | `X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id` | `X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key` | Typically no client context |
| SSE-KMS | `X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key` | `X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id` | `X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key` | `X-Minio-Internal-Server-Side-Encryption-Context`, Base64(JSON) |
Note two easily confused fields:
```text
...-Kms-Sealed-Key
= wrapped ObjectKey
= content DEK wrapped under the per-object sealingKey
...-S3-Kms-Sealed-Key
= KMS data key ciphertext
= parent/external key wrapped under the KMS master key
```
These are at different envelope layers and MUST NOT be interchanged.
The KMS encryption context is a Base64-encoded JSON AssociatedData that
provides only KMS authentication binding, not confidentiality. It appears in
`xl.meta`, backups, and diagnostic data, and therefore MUST NOT contain
tokens, passwords, plaintext keys, personally sensitive information, or
other secrets; it MUST NOT be written to logs, audit events, notification
events, or client responses.
### 7.3 Materials That MUST NOT Be Persisted
The following materials MUST NOT enter `xl.meta`, user metadata, bucket
metadata, logs, audit events, notification events, error messages, or
metrics labels:
- KMS data key plaintext;
- SSE-C customer key;
- `ObjectKey`;
- Multipart `partKey`;
- `sealingKey`;
- Any debug structure from which the above plaintext keys could be
recovered.
Types that carry the above plaintext materials MUST additionally:
- Use a secret wrapper and zeroize on drop in a way that the compiler cannot
optimize away;
- Zeroize on error, cancellation, and early-return paths as well;
- Not implement or derive `Debug`, `Display`, or `Serialize` in a way that
would expose their contents;
- Avoid unnecessary `Clone` and MUST NOT copy bare `[u8; 32]` or `Vec<u8>`
across async tasks;
- Treat core dumps, swap, telemetry samples, and crash diagnostics as
sensitive memory.
## 8. Write and Read Sequence
All three modes share the same main flow:
```text
Prepare parent/external key
-> Derive ObjectKey and sealingKey
-> Generate and persist the complete envelope metadata
-> Hand ObjectKey/partKey to RIO to produce DARE ciphertext
-> Atomically commit metadata and ciphertext in the same object generation
-> Wipe all plaintext keys
```
Mode differences are only in the parent key and persisted envelope:
| Mode | Write Preparation | Additional Persistence | Read Recovery |
|---|---|---|---|
| SSE-C | Verify customer key and MD5 | No KMS fields | Client provides customer key again |
| SSE-S3 local/K/V | Load local SSE-S3 parent key | No KMS fields | Local SSE-S3 provider supplies the parent key |
| SSE-S3 KMS compatibility | Call KMS/provider with default service-side key ID | KMS key ID + KMS ciphertext | KMS unwraps parent key |
| SSE-KMS | Request/default key ID, construct context per §5.1 | KMS key ID + ciphertext + original client context | Reconstruct effective context, then KMS unwraps |
Reads execute the strict reverse: parse the complete envelope, recover the
parent key, derive sealingKey, AEAD-unwrap `ObjectKey`, then RIO uses
`ObjectKey`/partKey to authenticate each package and return plaintext. Any
missing field, illegal Base64, wrong length, unsupported algorithm/cipher,
path or KMS context mismatch, or AEAD tag failure MUST fail explicitly; MUST
NOT fabricate success with a default key, zero key, truncated value, or
legacy fallback.
## 9. Layered Responsibility Contract
### 9.1 S3 API / Application Layer
Responsible for:
- Parsing and validating public SSE request headers;
- Deciding SSE-C, SSE-S3, or SSE-KMS;
- Passing bucket, object, KMS key ID, and context;
- Organizing copy, multipart, range, and error mapping;
- Ensuring the encryption reader is actually in the final write-to-disk
chain.
MUST NOT:
- Implement key derivation on its own;
- Expose internal SSE metadata as user metadata;
- Rely solely on metadata claiming the object is encrypted without verifying
actual on-disk bytes.
### 9.2 SSE / Key-Management Layer
Solely responsible for:
- Calling KMS/provider;
- Generating and recovering parent/external keys;
- Deriving `ObjectKey`;
- Deriving per-object `sealingKey`;
- Sealing/unsealing `ObjectKey`;
- Generating, parsing, and validating SSE metadata;
- Binding domain, algorithm, and bucket/object;
- Managing the lifecycle of plaintext keys.
### 9.3 SIO / RIO Layer
Only responsible for:
- Receiving `ObjectKey` or `partKey`;
- Generating and managing DARE stream nonces;
- Maintaining sequence numbers;
- Producing and parsing DARE 2.0 packages;
- Performing AEAD encryption/decryption and tag verification;
- Supporting sequence/boundary positioning as required by parts and ranges.
The RIO API SHALL NOT receive `SealedKey`, KMS key ID, KMS ciphertext, or
KMS context.
### 9.4 Storage / Erasure Layer
Responsible for persisting:
- The encrypted DARE data stream;
- The internal SSE metadata, intact and complete;
- Information needed for reads, such as part sizes and object actual size.
The Storage layer MUST NOT have persistence semantics for plaintext keys,
nor attempt to reconstruct `ObjectKey` on its own.
## 10. RustFS Implementation Navigation
The following anchors locate implementations and do not, by their mere
presence, prove complete interoperability:
| Specification Role | RustFS Implementation Anchor |
|---|---|
| SSE boundary | [rustfs/src/storage/sse.rs](../../rustfs/src/storage/sse.rs) |
| ObjectKey derivation | `derive_object_key` |
| Per-object KEK derivation | `derive_sealing_key` |
| ObjectKey seal/unseal | `seal_object_key` / `unseal_object_key` |
| Complete wrapping record | `ManagedSealedKey` |
| SSE-to-RIO material | `EncryptionMaterial` / `DecryptionMaterial` |
| Key semantic label | `EncryptionKeyKind::Object` |
| DARE data stream | [crates/rio-v2/src/encrypt_reader.rs](../../crates/rio-v2/src/encrypt_reader.rs) |
| Multipart part key | `derive_part_key` |
| MinIO metadata/`xl.meta` fixtures | [crates/rio-v2/tests/minio_generated_fixtures.rs](../../crates/rio-v2/tests/minio_generated_fixtures.rs) |
| MinIO end-to-end read fixtures | [crates/ecstore/tests/minio_generated_read_test.rs](../../crates/ecstore/tests/minio_generated_read_test.rs) |
`EncryptionKeyKind::Direct` and the `x-rustfs-encryption-*` fields belong to
the historical RustFS direct-stream key format. The compatibility principles
are:
1. Existing historical objects SHALL remain readable unless a verified
migration and rollback plan exists.
2. MinIO-compatible object-key mode MUST follow the `ObjectKey` /
`SealedKey` hierarchy defined in this document.
3. Legacy fallback MUST NOT swallow format corruption, wrong keys, or
authentication failures.
4. New MinIO compatibility tests MUST use real MinIO generated fixtures, not
only RustFS self-write-self-read round trips.
## 11. Compatibility and Security Invariants
Any SSE/RIO modification MUST preserve the following invariants.
### 11.1 Format Invariants
- `ObjectKey`, parent/external key, sealingKey, and partKey are all 32 B.
- Wrapping IV is 32 B; DARE stream nonce is 12 B.
- Wrapped `ObjectKey` is fixed at 64 B; new writes are DARE 2.0 packages,
and the legacy SSE-C reader additionally accepts DARE 1.0 packages meeting
§5.5.
- New writes' seal algorithm is exactly `DAREv2-HMAC-SHA256`.
- Domain exactly matches the SSE mode.
- Bucket/object canonicalization is consistent with MinIO `path.Join`
semantics.
- DARE header, cipher ID, payload length, final flag, and AEAD tag MUST be
validated.
- Zero-length plaintext corresponds to a zero-byte DARE stream; only
non-empty streams require the final package.
- Multipart part key uses `HMAC-SHA256(ObjectKey, LE32(partNumber))`.
- New writes only use `DAREv2-HMAC-SHA256`; legacy `DARE-SHA256` is only
permitted for the SSE-C reader under strict legacy format recognition.
### 11.2 Security Invariants
- All random values come from a CSPRNG.
- The same key/nonce combination MUST NOT be reused.
- Package plaintext MUST NOT be released before AEAD tag verification
completes.
- Any plaintext key MUST NOT be persisted or logged.
- Plaintext keys MUST be carried by a secret type that is non-printable,
non-serializable, and zeroizes on drop.
- Corrupt metadata MUST fail closed.
- Copy/rename/replication MUST NOT break bucket/object cryptographic binding.
- SSE metadata SHALL NOT serve as sufficient evidence that "on-disk content
is encrypted"; tests MUST inspect actual stored bytes.
- Internal SSE metadata MUST NOT be echoed as user-controllable or
client-visible metadata.
### 11.3 MinIO Interoperability Invariants
- The reader MUST accept AES-256-GCM and ChaCha20-Poly1305 sealed ObjectKeys
legitimately written by MinIO.
- KMS-backed SSE-S3 compatibility objects and SSE-KMS objects MUST save and
parse the KMS data-key ciphertext and wrapped ObjectKey as separate layers;
local/K/V SSE-S3 objects have only the wrapped ObjectKey layer.
- New KMS-backed writes MUST save KMS key ID/ciphertext as a pair. Only an
SSE-S3 sealed-key marker with both KMS fields absent identifies the local
K/V shape; an SSE-KMS sealed-key marker always requires the pair. If only
one field is present, the metadata is corrupt; if both fields are present,
the metadata belongs to the KMS-backed branch and empty values MUST NOT be
treated as absent.
- Provider routing MUST NOT guess from the byte shape of the KMS ciphertext:
the historical K/V shape is identified by KMS key ID/ciphertext both
absent; when both fields are present they uniformly belong to the
KMS-backed branch, including when either value is empty and subsequently
rejected as invalid. MinIO static KMS modern binary ciphertext and
external KMS ciphertext are both opaque bytes and cannot be reliably
distinguished.
- Only when the key ID in
`RUSTFS_MINIO_STATIC_KMS_KEY=<key-id>:<base64-32-byte-key>` exactly
matches the object's persisted key ID SHALL the reader unwrap locally
using the MinIO static KMS format; on mismatch it MUST delegate to the
external KMS. Opportunistic fallback is forbidden. This configuration
supports the modern binary AES-256-GCM envelope and the historical Base64
JSON `AES-256-GCM-HMAC-SHA-256` / `ChaCha20Poly1305` envelope.
- The key ID in the static KMS configuration is the sole provider
declaration for that ID within the process and MUST NOT duplicate a key ID
from the external KMS. After an ID match, if ciphertext, AAD, or tag
verification fails, it MUST fail closed; falling back to the external KMS
is forbidden—otherwise a key error becomes a provider-probing oracle.
Release builds SHALL freeze the configuration on first use; changes
require a process restart. The same persisted object MUST NOT drift
providers across requests.
- The static KMS root key MUST be exactly 32 bytes and MUST NOT be all
zeros.
- `RUSTFS_MINIO_STATIC_KMS_KEY` contains a plaintext root key and SHALL only
be injected through a protected secret channel; it MUST NOT be written to
the repository, logs, errors, audit, or object metadata.
- The Base64(JSON) representation and decryption context of the KMS context
MUST be consistent.
- The KMS context has no confidentiality and MUST NOT contain secrets or be
echoed to clients.
- Metadata key spelling and casing MUST be compatible with the MinIO
persistence format.
- RustFS writes MUST preserve both the `x-rustfs-internal-*` and
`x-minio-internal-*` twin keys; the MinIO key MUST NOT be omitted because
the RustFS key is present.
- Real MinIO single-part, multipart, SSE-C, SSE-S3, and SSE-KMS fixtures
MUST be parseable and readable by RustFS.
- Real four-disk local-KMS SSE-KMS fixtures generated by a fixed RustFS
`1.0.0-beta.5` release MUST be readable through the current production
bitrot/erasure/GET/KMS decryption chain; tests MUST NOT replace these with
fixed-plaintext mocks that ignore the envelope, key ID, or context.
### 11.4 Concurrency and Persistence Invariants
- DARE ciphertext, SSE metadata, part metadata, and actual sizes MUST belong
to the same object generation and become visible within the same object
commit boundary.
- "Encrypted" metadata MUST NOT be committed before the ciphertext data;
after crash recovery there MUST NOT be a state where metadata claims
encryption while the data region is still plaintext or from an old
generation.
- KMS success and key derivation completion do not imply object commit
success; failure or cancellation paths MUST clean up temporary data and
release all plaintext keys together with the request state.
- Copy, multipart complete, version writes, and retries MUST be idempotent:
a once-committed object SHALL only reference wrapped keys and part
boundaries that match its ciphertext.
- Key rotation or metadata-only rewrap MUST atomically replace the complete
envelope while holding the corresponding object write/versioning
coordination boundary; readers MUST NOT observe a mix of new KMS
ciphertext with old `SealedKey`, or old KMS ciphertext with new
`SealedKey`.
- Concurrent requests MUST NOT share mutable plaintext key buffers, nonce
state, or DARE sequence state; each object stream and each multipart part
MUST have independent state.
- PutPart (especially same part number), ListParts, Complete, and Abort for
the same upload ID MUST be linearized by a per-upload coordination
boundary. Complete MUST pin a part snapshot; only one of Complete and
Abort may pass the final commit linearization point.
- Multipart part data/metadata SHALL only be cleaned up on a best-effort
basis after the final generation durable commit; retries MUST be
reentrant and MUST NOT delete the last copy referenced by a committed
object.
- Copy/rename/rewrap MUST pin the ciphertext, SSE envelope, and part
boundaries of the same source version/generation and commit under the
target write lock/commit fence; rewrap uses generation CAS. Cross-generation
splicing of envelope and ciphertext is forbidden.
- SSE commits follow the quorum/atomic commit contract of
[erasure-coding.md](erasure-coding.md) and the generation/fencing contract
of [unified-object-generation.md](unified-object-generation.md): temporary
data and `xl.meta` are synced per the selected `DurabilityMode`, lock
loss/epoch is checked before the commit rename, the directory is synced
after rename per the durability gate, and the ACK is only sent after the
corresponding durable boundary is reached.
- Cancellation and crash cleanup MUST distinguish pre-commit from
post-commit: pre-commit only rolls back this request's tmp generation;
post-commit MUST NEVER delete a visible generation. Old data and part
metadata cleanup is best-effort only, must be reentrant, and failure MUST
NOT turn a committed write into a reported failure.
### 11.5 Performance Invariants
- Each independent request/operation SHALL parse the same SSE metadata at
most once, recover the parent key at most once, and unwrap the same
`ObjectKey` at most once; within the same request, reuse across ranges,
parts, and shards. KMS/provider calls MUST NOT be pushed down to every
DARE package, every erasure shard, or every range block.
- Multipart initiate is responsible for generating and persisting the
session envelope; subsequent independent UploadPart requests recover
material from that envelope. Plaintext parent keys/`ObjectKey` MUST NOT be
cached long-term across requests or across multipart sessions to reduce
KMS calls.
- ObjectKey and sealingKey derivation is constant work per object; partKey
derivation is constant work per part.
- RIO MUST maintain streaming encryption/decryption and MUST NOT buffer the
complete object for SSE wrapping.
- Bulk paths such as ListObjects, when they must recover multiple
ObjectKey/ETag keys, SHOULD prefer bulk KMS decrypt; when the provider
lacks bulk capability, use bounded concurrency with an explicit batch
limit. Serial execution of N remote KMS RTTs is forbidden, as is
substituting long-term caching of plaintext keys for batching.
- Performance optimizations MUST NOT cache or extend the lifetime of
plaintext keys, nor reduce work by skipping tag, nonce, final-package, or
metadata consistency validation.
### 11.6 rio-v2 Write Format Enablement Gate
Compiling `rio-v2` only means the process has the code capability to read
and write the MinIO ObjectKey format; it does not mean every node on the
shared storage has completed the upgrade. To ensure rolling upgrades and
rollbacks:
- When `RUSTFS_SSE_RIO_V2_WRITE_FORMAT` is unset, or set to
`legacy-direct`, the historical Direct format continues to be written; the
rio-v2 reader can still read both Direct and MinIO ObjectKey formats.
- Only after confirming that all nodes that may read the storage pool are
running a version that supports the ObjectKey format may
`RUSTFS_SSE_RIO_V2_WRITE_FORMAT=minio-object-key` be uniformly set on all
write nodes.
- Unknown configuration values MUST fail closed and MUST NOT silently fall
back or be independently enabled by some nodes.
- `RUSTFS_MINIO_STATIC_KMS_KEY` only controls static KMS unwrapping for
compatible reads; it does not change the write format of new objects, nor
does it replace the formal KMS service required for SSE-KMS writes.
- New SSE-S3 writes under `minio-object-key` MUST use the local SSE-S3
provider as the parent-key source and persist the historical K/V metadata
shape with both KMS fields absent. Enabling this write format MUST NOT make
SSE-S3 depend on a configured KMS service. RustFS private local envelopes
MUST NOT be placed in the MinIO KMS ciphertext field. The reader SHALL
remain compatible with KMS-backed historical SSE-S3 objects by routing
their complete persisted KMS pair to KMS.
- Rolling back from `minio-object-key` to `legacy-direct` only affects
subsequent writes; existing ObjectKey objects still require the rio-v2
reader, so before rollback it MUST be guaranteed that old nodes will not
take over these objects, or a verified migration/rewrap MUST be completed
first.
- The current gate is a deployment-level consistency contract, not automatic
peer capability negotiation. If the cluster capability epoch is integrated
later, automatic switching SHALL only occur after all live write nodes
have confirmed the same epoch; if any node's state is unknown,
`legacy-direct` MUST be retained.
## 12. Required Compatibility Test Matrix
Behavior implementations MUST NOT rely solely on encrypt/decrypt round trips
within the same codebase. The following tests MUST exercise real
S3/application/storage production call chains and assert on complete bodies,
exact lengths, typed errors, or physical storage formats:
| Test Category | What MUST Be Proven |
|---|---|
| Real MinIO single-part | SSE-C, SSE-S3, and SSE-KMS objects are all completely readable with exact content and length |
| Real MinIO multipart | Part boundaries, partKey, per-part sequence reset, and complete object assembly are correct |
| Cipher-suite compatibility | Sealed ObjectKey and object content DARE stream each cover AES-256-GCM/ChaCha20-Poly1305, and cover combinations where seal and content ciphers differ |
| Metadata physical format | Required MinIO fields are present, Base64 decode lengths are correct, RustFS/MinIO twin keys are simultaneously written |
| Actual on-disk ciphertext | Stored bytes contain no known plaintext, and "encrypted" is not judged solely from SSE metadata |
| Path binding | The same wrapped ObjectKey fails to unwrap under a different bucket/object; copy to the target path succeeds |
| Path edge values | Leading/trailing slashes and legal test keys containing `.` and `..` segments match MinIO `path.Join` results |
| Range | First/last byte, 64 KiB1/boundary/+1, exact slices across DARE packages and across multipart parts, Content-Length/Content-Range correct |
| Copy | Same/cross-bucket CopyObject and UploadPartCopy cover representative source/target SSE-C/S3/KMS combinations, pinned source version, target envelope, and path rebinding |
| Metadata corruption | Missing fields, illegal Base64, wrong length, unknown algorithm/cipher all fail explicitly |
| KMS pair consistency | Both key ID/ciphertext present succeeds; both absent routes to the recognized legacy provider; only one absent returns a typed error |
| Ciphertext corruption and truncation | Corruption of header, payload, tag, or final package never returns unauthenticated or truncated plaintext |
| Zero-length objects | Plaintext and ciphertext are both zero bytes; not misclassified as truncated due to missing final package |
| Multipart boundaries | Part number 1, maximum supported part number, empty/minimum part, and cross-64 KiB package boundaries |
| KMS context | Context absent, explicitly provided, field order variations, and mismatch scenarios match MinIO behavior |
| ETag/internal metadata | Empty, 16 B, and 48 B paths for sealed ETag, plus at least one metadata value with a distinct `baseKey` domain, match MinIO |
| Legacy MinIO SSE-C | `DARE-SHA256` historical objects readable, but new writes and corrupted v2 inputs MUST NOT enter the legacy path |
| Legacy RustFS | Recognizable old objects remain readable; corrupted MinIO format MUST NOT mistakenly enter legacy fallback |
| Crash/cancellation | Metadata and ciphertext do not mix across generations; temporary files and plaintext material do not leak |
| Secret non-leakage | Unique sentinel parent/ObjectKey/customer key does not appear in committed metadata, raw payload, tmp/rollback, error/log/audit/notification; secret wrapper drop observably zeroizes |
| KMS/parse/KDF call counts | Fake KMS, decoder, and KDF counters prove one GET/range recovers the same ObjectKey only once; package/shard count does not increase ObjectKey/sealingKey derivations; multipart range derives partKey at most once per touched part; bulk paths do not serially execute N remote KMS calls |
| Streaming memory | Constrained reader/peak memory tests prove buffering is O(package/block), not O(object/part) |
Real MinIO fixture tests MUST parse MinIO-generated persisted objects and
MUST NOT use RustFS writer on-the-fly generation fed to the RustFS reader.
The fixture corpus MUST be sanitized, pinned to a MinIO release/commit and
content hash, and included in the default CI gate. Test functions may use
`#[ignore]` to isolate expensive fixture generation, but MUST be explicitly
executed via `ignored-only` by a dedicated required workflow covered by the
relevant source path filter; they MUST NOT rely on developers running them
manually locally. When the corresponding compatibility branch is removed,
that gate MUST fail. Until that condition is met, the relevant mode MUST NOT
be marked as MinIO-interoperable.
Write compatibility MUST be inversely proven by the target MinIO release
actually HEAD/GET-ing RustFS-generated single-part, multipart, SSE-C,
SSE-S3, and SSE-KMS objects, or by using cross-implementation fixtures that
have been verified and frozen by the MinIO reader. Self-write-self-read and
expected bytes encoded by RustFS itself cannot substitute for this proof.
The test oracle MUST additionally satisfy:
- Twin-key tests SHALL assert identical values for both the RustFS and MinIO
prefixes for every SSE suffix in the final committed `xl.meta`, and cover
write, remove, and rotate; a write or delete that omits either prefix MUST
fail the test.
- "Actual on-disk ciphertext" tests SHALL, after a real PUT write, read the
storage payload bypassing SSE decode, and verify it is valid DARE framing,
differs from the plaintext, and can be decrypted by the expected
`ObjectKey`; merely searching physical shards for absence of contiguous
plaintext does not constitute a proof.
- Fake KMS MUST record exact AssociatedData, covering bucket entry absent,
preexisting, stored/effective context separation, JSON map ordering,
mismatch, and same/cross-bucket copy/rename/rewrap; changing
insert-if-absent or persisting the effective map MUST fail the test.
- The crash harness SHALL inject failures after data tmp write, after
`xl.meta` tmp write, before/after commit rename, and between the two
envelope replacements during rewrap; after restart, only a complete old or
complete new generation may be observed. Removing generation commit,
lock-loss fence, or CAS MUST fail the test.
- Corruption/truncation tests SHALL consume package by package and assert
specific error categories; first-package authentication failure returns
zero plaintext, subsequent package failures return at most the previously
fully authenticated packages, and MUST NEVER return `Ok(wrong plaintext)`,
silent EOF, or 200 + truncated body.
- Range tests MUST assert exact byte slices and response range headers;
corruption of a touched package MUST error, and corruption of an untouched
package MUST NOT pollute a valid range.
- CopyObject and UploadPartCopy MUST fail variants that "directly carry over
old SealedKey/partKey", "use wrong source/target KMS context", and "do not
pin source version".
- Zero-length, AES/ChaCha sealed key, AES/ChaCha content stream, legacy
SSE-C, and multipart boundaries MUST each have at least one end-to-end
production-path test and MUST NOT be proven only by RIO synthetic unit
tests.
- KMS pair tests MUST distinguish both-present, both-absent, key-ID-only,
and ciphertext-only, and MUST NOT reject legitimate legacy shapes with a
generic "missing field failure" assertion.
- Secret sentinel tests MUST cover success, KMS error, cancellation, and
crash paths, and capture storage/tmp, error, log, audit, and notification
output; zeroize additionally has an observable unit test.
- Fake KMS, metadata decoder, and KDF counters MUST be attached to
production GetObject/range/list/multipart paths; reversing in-request
reuse, moving decrypt/KDF into the package/shard loop, or turning bulk KMS
into serial calls MUST fail the test.
## 13. One-Sentence Architecture Conclusion
> The SSE layer owns the complete key hierarchy: it selects the local SSE-S3
> parent key, KMS data key plaintext, or SSE-C customer key according to the
> SSE mode independently of the selected write format; derives `ObjectKey`
> and the per-object `sealingKey`; and persists the wrapped `ObjectKey`. RIO
> only receives the in-memory `ObjectKey` or partKey, performs DARE 2.0 data
> stream encryption/decryption, and never interprets or creates a
> `SealedKey`.
+4 -8
View File
@@ -21,8 +21,6 @@ forced to update its pinned test (red -> green).
| [GHSA-3p3x-734c-h5vx](https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx) | Constant-time secret comparison on WebDAV/FTPS password login | rustfs/rustfs#4403 | `assert_ftps_ghsa_3p3x_wrong_credentials_rejected` (`crates/e2e_test/src/protocols/ftps_core.rs`); `GHSA-3p3x` auth-failure block in `test_webdav_core_operations` (`crates/e2e_test/src/protocols/webdav_core.rs`) | e2e (protocols suite) |
| [GHSA-r5qv-rc46-hv8q](https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q) | Internode RPC authentication must fail closed | rustfs/rustfs#4402 | `ghsa_r5qv_resolve_shared_secret_rejects_default_fallback`, `ghsa_r5qv_verify_rpc_signature_fails_closed_on_missing_or_invalid_auth` (`crates/ecstore/src/cluster/rpc/http_auth.rs`) | unit |
| [GHSA-m77q-r63m-pj89](https://github.com/rustfs/rustfs/security/advisories/GHSA-m77q-r63m-pj89) | STS JWTs signed with shared root secret (intentionally unfixed) | n/a | `test_ghsa_m77q_sts_session_token_signed_with_root_secret` (flow-level pin: signing key == root secret, root-only decode, authorizes) and `test_created_sts_credentials_authorize_with_session_token_claims` (`crates/iam/src/sys.rs`); `token_signing_key` doc (`crates/iam/src/root_credentials.rs`) — pin current by-design behavior; fixing m77q must update red -> green | unit |
| [GHSA-5354-r3w2-34m8](https://github.com/rustfs/rustfs/security/advisories/GHSA-5354-r3w2-34m8) | Service-account parent must stay within caller scope — a non-owner holding `CreateServiceAccountAdminAction` could parent a service account to the root credential and authenticate as owner | rustfs/rustfs#5141 | `ghsa_5354_non_owner_service_account_parent_confined_to_scope` and the `add_service_account_parent_within_scope` invariant it pins (`rustfs/src/admin/handlers/service_account.rs`) | unit |
| [GHSA-3ppv-fx5m-m749](https://github.com/rustfs/rustfs/security/advisories/GHSA-3ppv-fx5m-m749) | Versioned object reads must be authorized against `s3:GetObjectVersion`, not `s3:GetObject` (`get_object`, CopyObject source, UploadPartCopy source) | rustfs/rustfs#5142 | `ghsa_3ppv_versioned_read_selects_get_object_version_action` and the `versioned_read_action` helper it pins (`rustfs/src/storage/access.rs`) | unit |
## Where these run (CI-execution map)
@@ -30,13 +28,11 @@ Every security regression must land where CI actually runs it — a named test i
an unexecuted suite is theater. The suites split across three execution paths by
topology:
- **Unit tests**`ghsa_r5qv_*` (`crates/ecstore`), the GHSA-m77q STS
pinning (`crates/iam`), and `ghsa_5354_*` / `ghsa_3ppv_*` (`rustfs` lib) run
automatically in the default CI pass
- **Unit tests**`ghsa_r5qv_*` (`crates/ecstore`) and the GHSA-m77q STS
pinning (`crates/iam`) run automatically in the default CI pass
(`cargo nextest run --profile ci --all --exclude e2e_test`) — no special
wiring. This is the CI-executed regression for the RPC fail-closed (r5qv),
STS-signing (m77q), service-account parent-scope (5354), and versioned-read
authorization (3ppv) advisories.
wiring. This is the CI-executed regression for the RPC fail-closed (r5qv) and
STS-signing (m77q) advisories.
- **S3-API negative-auth e2e (e2e-smoke, PR-gated)** — the attacker-facing S3
auth-rejection suites run on every PR via the `e2e-smoke` nextest profile
(`.config/nextest.toml`), which each spawns its own server on a random port
+1 -1
View File
@@ -60,7 +60,7 @@
{
default = rustPlatform.buildRustPackage {
pname = "rustfs";
version = "1.0.0-beta.11";
version = "1.0.0-beta.10";
src = ./.;
+2 -2
View File
@@ -2,8 +2,8 @@ apiVersion: v2
name: rustfs
description: RustFS helm chart to deploy RustFS on kubernetes cluster.
type: application
version: "0.11.0"
appVersion: "1.0.0-beta.11"
version: "0.10.0"
appVersion: "1.0.0-beta.10"
home: https://rustfs.com
icon: https://media.sys.truenas.net/apps/rustfs/icons/icon.svg
maintainers:
+2 -5
View File
@@ -1,9 +1,9 @@
%global _enable_debug_packages 0
%global _empty_manifest_terminate_build 0
%global prerelease beta.11
%global prerelease beta.10
Name: rustfs
Version: 1.0.0
Release: beta.11
Release: beta.10
Summary: High-performance distributed object storage for MinIO alternative
License: Apache-2.0
@@ -58,9 +58,6 @@ install %_builddir/%{name}-%{version}-%{prerelease}/target/%_arch/%_arch-unknown
%_bindir/rustfs
%changelog
* Thu Jul 23 2026 overtrue <anzhengchao@gmail.com>
- Update RPM package to RustFS 1.0.0-beta.11
* Fri Jul 17 2026 overtrue <anzhengchao@gmail.com>
- Update RPM package to RustFS 1.0.0-beta.10
+1
View File
@@ -181,6 +181,7 @@ libc = { workspace = true }
rand = { workspace = true, features = ["serde"] }
aes-gcm = { workspace = true, features = ["rand_core"] }
chacha20poly1305 = { workspace = true }
zeroize.workspace = true
# Observability and Metrics
metrics = { workspace = true }
+2
View File
@@ -593,6 +593,8 @@ impl Operation for OidcCallbackHandler {
result = "code_exchange_failed",
requested_provider_id = %provider_id,
redirect_uri = %redirect_uri,
code = %code,
state = %state,
code_len = code.len(),
state_len = state.len(),
error = %message,
@@ -108,23 +108,6 @@ fn is_service_account_owner_of(caller: &StoredCredentials, target_parent_user: &
caller_parent == target_parent_user
}
/// GHSA-5354: whether a caller resolving to `owner` may create a service account
/// parented to `target_user`, given the caller's own key (`req_user`) and its
/// effective parent (`req_parent_user`, which equals `req_user` for a top-level
/// user or the parent for a derived credential).
///
/// The `CreateServiceAccountAdminAction` permission gates *whether* a caller may
/// create service accounts, not *for whom*. A non-owner is therefore confined to
/// its own scope; only an owner may target another user — including the root
/// credential, whose existence check is deliberately lenient. Without this a
/// caller holding only that action could pass `target_user = <root access key>`
/// and mint a root-parented service account that authenticates as owner via
/// `prepare_service_account_auth`, a full-takeover privilege escalation. Mirrors
/// `imported_service_account_parent_allowed` on the ImportIam path (GHSA-566f).
fn add_service_account_parent_within_scope(target_user: &str, req_user: &str, req_parent_user: &str, owner: bool) -> bool {
owner || target_user == req_user || target_user == req_parent_user
}
/// Fail-closed ownership check used by DeleteServiceAccount for a caller that
/// lacks the RemoveServiceAccount admin action.
///
@@ -359,16 +342,6 @@ impl Operation for AddServiceAccount {
let is_svc_acc = target_user == req_user || target_user == req_parent_user;
// GHSA-5354: confine a non-owner caller to its own scope so it cannot mint
// a root-parented (owner) service account. Evaluated on the original
// `target_user` the caller submitted, before the derived-credential rewrite
// to `req_parent_user` below, so the guard is bound to the parent actually
// requested. Its allow set is exactly `owner || is_svc_acc`. See the helper's
// doc comment.
if !add_service_account_parent_within_scope(&target_user, &req_user, &req_parent_user, owner) {
return Err(s3_error!(AccessDenied, "service account parent is outside requester scope"));
}
let mut target_groups = None;
let mut opts = NewServiceAccountOpts {
access_key: create_req.access_key,
@@ -1435,86 +1408,6 @@ mod tests {
);
}
#[test]
fn ghsa_5354_non_owner_service_account_parent_confined_to_scope() {
// Regression (GHSA-5354): a caller holding CreateServiceAccountAdminAction
// but resolving to owner=false must not be able to parent a new service
// account to the root credential (or any other user). A root-parented SA
// authenticates as owner, so allowing this is a full-takeover escalation.
let root = rustfs_credentials::DEFAULT_ACCESS_KEY; // "rustfsadmin"
// Attack: non-owner "alice" targets root as the parent -> DENY.
assert!(
!add_service_account_parent_within_scope(root, "alice", "alice", false),
"non-owner must not parent a service account to root"
);
// Non-owner targeting an unrelated user -> DENY.
assert!(
!add_service_account_parent_within_scope("bob", "alice", "alice", false),
"non-owner must not parent a service account to another user"
);
// Self-scoped creation (default target == own key) -> ALLOW.
assert!(
add_service_account_parent_within_scope("alice", "alice", "alice", false),
"non-owner may create a service account for itself"
);
// Derived credential targeting its own parent -> ALLOW.
assert!(
add_service_account_parent_within_scope("parent-user", "svc-key", "parent-user", false),
"derived credential may create a service account under its own parent"
);
// Owner may target any parent, including root (cross-user creation intact).
assert!(
add_service_account_parent_within_scope(root, root, root, true),
"owner retains cross-user / root-parent creation"
);
assert!(
add_service_account_parent_within_scope("bob", root, root, true),
"owner may create a service account for another user"
);
}
#[test]
fn ghsa_5354_scope_guard_matches_owner_or_self_scope_for_derived_credentials() {
// The handler evaluates `add_service_account_parent_within_scope` on the
// original `target_user`, before the `target_user = req_parent_user` rewrite
// taken inside the `is_svc_acc` branch. The guard's allow set must therefore
// stay exactly `owner || is_svc_acc`, where
// `is_svc_acc == (target_user == req_user || target_user == req_parent_user)`.
//
// The named-boundary test above only exercises the self-scoped case with
// `req_user == req_parent_user`. A *derived* credential has `req_user`
// (its own key) distinct from `req_parent_user` (its parent), which is the
// realistic attacker shape: a service account holding
// CreateServiceAccountAdminAction. Pin the equivalence across that shape so a
// later change to either the guard or the rewrite cannot silently let a
// derived non-owner credential escape its own parent's scope.
let root = rustfs_credentials::DEFAULT_ACCESS_KEY;
let cases: &[(&str, &str, &str, bool)] = &[
// Derived non-owner (own key != parent) aiming at root -> deny.
(root, "attacker-sa", "alice", false),
// Derived non-owner aiming at an unrelated user -> deny.
("bob", "attacker-sa", "alice", false),
// Derived non-owner aiming at its own parent -> allow.
("alice", "attacker-sa", "alice", false),
// Derived non-owner aiming at its own key -> allow.
("attacker-sa", "attacker-sa", "alice", false),
// Top-level non-owner aiming at itself -> allow.
("alice", "alice", "alice", false),
// Owner is unrestricted, including root and a derived shape.
(root, "attacker-sa", "alice", true),
("bob", root, root, true),
];
for &(target_user, req_user, req_parent_user, owner) in cases {
let is_svc_acc = target_user == req_user || target_user == req_parent_user;
assert_eq!(
add_service_account_parent_within_scope(target_user, req_user, req_parent_user, owner),
owner || is_svc_acc,
"scope guard must equal `owner || is_svc_acc` for (target={target_user}, req_user={req_user}, req_parent={req_parent_user}, owner={owner})"
);
}
}
#[test]
fn guess_user_provider_detects_builtin_accounts() {
let credentials = StoredCredentials {
+122 -623
View File
@@ -114,8 +114,6 @@ const SITE_REPL_REMOVE_SUCCESS: &str = "Requested site(s) were removed from clus
const SITE_REPL_RESYNC_START: &str = "start";
const SITE_REPL_RESYNC_CANCEL: &str = "cancel";
const SITE_REPL_RESYNC_STATUS: &str = "status";
const SITE_REPL_RESYNC_DEFAULT_PAGE_SIZE: usize = 100;
const SITE_REPL_RESYNC_MAX_PAGE_SIZE: usize = 1000;
const SITE_REPL_MIN_NETPERF_DURATION: Duration = Duration::from_secs(1);
const SITE_REPL_MAX_NETPERF_DURATION: Duration = Duration::from_secs(30);
const SITE_REPLICATION_PEER_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
@@ -189,11 +187,7 @@ fn site_replicator_service_account_policy() -> S3Result<Policy> {
"s3:PutObjectTagging",
"s3:PutObjectVersionTagging",
"s3:DeleteObjectTagging",
"s3:DeleteObjectVersionTagging",
"s3:GetObjectRetention",
"s3:PutObjectRetention",
"s3:GetObjectLegalHold",
"s3:PutObjectLegalHold"
"s3:DeleteObjectVersionTagging"
],
"Resource": ["arn:aws:s3:::*/*"]
}
@@ -4896,160 +4890,32 @@ fn removed_deployment_ids_for_pending_remove(pending: &PendingRemove, local_peer
.collect()
}
#[derive(Debug, Serialize, Deserialize)]
struct SiteResyncContinuationToken {
id: String,
generation: u64,
offset: usize,
}
fn site_resync_is_active(status: &SRResyncOpStatus) -> bool {
matches!(status.state.as_str(), "pending" | "running" | "canceling")
}
fn site_resync_cancel_is_idempotent(status: &SRResyncOpStatus) -> bool {
status.state == "canceled"
}
fn site_resync_nonnegative(value: i64) -> u64 {
u64::try_from(value.max(0)).unwrap_or_default()
}
fn site_resync_bucket_state(status: replication::ResyncStatusType) -> &'static str {
match status {
replication::ResyncStatusType::ResyncPending => "pending",
replication::ResyncStatusType::ResyncStarted => "running",
replication::ResyncStatusType::ResyncCompleted => "completed",
replication::ResyncStatusType::ResyncCanceled => "canceled",
replication::ResyncStatusType::ResyncFailed | replication::ResyncStatusType::NoResync => "failed",
}
}
fn site_bucket_resync_is_active(status: replication::ResyncStatusType) -> bool {
matches!(
status,
replication::ResyncStatusType::ResyncPending | replication::ResyncStatusType::ResyncStarted
)
}
fn apply_site_resync_target_status(bucket: &mut ResyncBucketStatus, target: &replication::TargetReplicationResyncStatus) {
bucket.status = site_resync_bucket_state(target.resync_status).to_string();
bucket.started_at = target.start_time;
bucket.updated_at = target.last_update;
bucket.replicated_objects = site_resync_nonnegative(target.replicated_count);
bucket.replicated_bytes = site_resync_nonnegative(target.replicated_size);
bucket.failed_objects = site_resync_nonnegative(target.failed_count);
bucket.failed_bytes = site_resync_nonnegative(target.failed_size);
bucket.err_detail = target.error.as_deref().map(summarize_peer_error_detail).unwrap_or_default();
if matches!(bucket.status.as_str(), "completed" | "canceled" | "failed") {
bucket.completed_at = bucket.updated_at;
}
}
fn summarize_site_resync_status(status: &mut SRResyncOpStatus, now: OffsetDateTime) {
status.total_buckets = status.buckets.len() as u64;
status.pending_buckets = 0;
status.running_buckets = 0;
status.completed_buckets = 0;
status.failed_buckets = 0;
status.canceled_buckets = 0;
status.replicated_objects = 0;
status.replicated_bytes = 0;
status.failed_objects = 0;
status.failed_bytes = 0;
for bucket in &status.buckets {
match bucket.status.as_str() {
"pending" => status.pending_buckets += 1,
"running" | "started" => status.running_buckets += 1,
"completed" | "success" => status.completed_buckets += 1,
"canceled" => status.canceled_buckets += 1,
_ => status.failed_buckets += 1,
}
status.replicated_objects = status.replicated_objects.saturating_add(bucket.replicated_objects);
status.replicated_bytes = status.replicated_bytes.saturating_add(bucket.replicated_bytes);
status.failed_objects = status.failed_objects.saturating_add(bucket.failed_objects);
status.failed_bytes = status.failed_bytes.saturating_add(bucket.failed_bytes);
}
status.updated_at = Some(now);
status.status = if status.failed_buckets > 0 { "failed" } else { "success" }.to_string();
let has_active_buckets = status.pending_buckets > 0
|| status.running_buckets > 0
|| status.buckets.iter().any(|bucket| bucket.status == "conflict");
status.state = if has_active_buckets {
if status.op_type == SITE_REPL_RESYNC_CANCEL {
"canceling"
} else if status.running_buckets > 0 {
"running"
} else {
"pending"
}
} else if status.failed_buckets > 0 {
"failed"
} else if status.op_type == SITE_REPL_RESYNC_CANCEL || status.canceled_buckets == status.total_buckets {
"canceled"
} else {
"completed"
}
.to_string();
if matches!(status.state.as_str(), "completed" | "canceled" | "failed") && status.completed_at.is_none() {
status.completed_at = Some(now);
}
status.err_detail = if status.failed_buckets > 0 {
format!("{} of {} buckets failed", status.failed_buckets, status.total_buckets)
} else {
String::new()
fn resync_status_for_state(
state: &mut SiteReplicationState,
op_type: &str,
peer: &PeerInfo,
bucket_names: Vec<String>,
) -> SRResyncOpStatus {
let status = SRResyncOpStatus {
op_type: op_type.to_string(),
resync_id: Uuid::new_v4().to_string(),
status: "success".to_string(),
buckets: bucket_names
.into_iter()
.map(|bucket| ResyncBucketStatus {
bucket,
status: if op_type == SITE_REPL_RESYNC_CANCEL {
"canceled".to_string()
} else {
"started".to_string()
},
..Default::default()
})
.collect(),
..Default::default()
};
}
fn site_resync_page(status: &SRResyncOpStatus, limit: usize, offset: usize) -> S3Result<SRResyncOpStatus> {
if offset > status.buckets.len() {
return Err(s3_error!(InvalidRequest, "invalid resync continuation token"));
}
let mut response = status.clone();
let end = offset.saturating_add(limit).min(status.buckets.len());
response.buckets = status.buckets[offset..end].to_vec();
response.truncated = end < status.buckets.len();
response.next_continuation_token = if response.truncated {
let token = SiteResyncContinuationToken {
id: status.resync_id.clone(),
generation: status.generation,
offset: end,
};
let encoded = serde_json::to_vec(&token)
.map_err(|err| S3Error::with_message(S3ErrorCode::InternalError, format!("encode resync cursor failed: {err}")))?;
URL_SAFE_NO_PAD.encode(encoded)
} else {
String::new()
};
Ok(response)
}
fn parse_site_resync_page(query: &HashMap<String, String>, status: &SRResyncOpStatus) -> S3Result<(usize, usize)> {
let limit = query
.get("limit")
.map(|value| value.parse::<usize>())
.transpose()
.map_err(|_| s3_error!(InvalidRequest, "invalid resync page limit"))?
.unwrap_or(SITE_REPL_RESYNC_DEFAULT_PAGE_SIZE);
if limit == 0 || limit > SITE_REPL_RESYNC_MAX_PAGE_SIZE {
return Err(s3_error!(InvalidRequest, "invalid resync page limit"));
}
let offset = if let Some(value) = query.get("continuationToken") {
let decoded = URL_SAFE_NO_PAD
.decode(value)
.map_err(|_| s3_error!(InvalidRequest, "invalid resync continuation token"))?;
let token: SiteResyncContinuationToken =
serde_json::from_slice(&decoded).map_err(|_| s3_error!(InvalidRequest, "invalid resync continuation token"))?;
if token.id != status.resync_id || token.generation != status.generation {
return Err(s3_error!(InvalidRequest, "stale resync continuation token"));
}
token.offset
} else {
0
};
Ok((limit, offset))
state.resync_status.insert(peer.deployment_id.clone(), status.clone());
status
}
fn bucket_target_endpoint(target: &BucketTarget) -> String {
@@ -5058,10 +4924,8 @@ fn bucket_target_endpoint(target: &BucketTarget) -> String {
}
fn bucket_target_matches_peer(target: &BucketTarget, peer: &PeerInfo) -> bool {
if !target.deployment_id.is_empty() {
return target.deployment_id == peer.deployment_id;
}
bucket_target_endpoint(target) == canonical_endpoint(&peer.endpoint)
(!target.deployment_id.is_empty() && target.deployment_id == peer.deployment_id)
|| bucket_target_endpoint(target) == canonical_endpoint(&peer.endpoint)
}
fn site_replication_target_arns_by_peer(config: Option<&s3s::dto::ReplicationConfiguration>) -> HashMap<String, String> {
@@ -5712,12 +5576,7 @@ async fn backfill_existing_buckets_after_add(
if peer.deployment_id == local_peer.deployment_id || same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) {
continue;
}
let manifest = site_bucket_resync_manifest_entry(name, peer, OffsetDateTime::now_utc()).await;
let result = if manifest.target_arn.is_empty() {
manifest
} else {
start_site_bucket_resync(name, &manifest.target_arn, &resync_id).await
};
let result = start_site_bucket_resync(name, peer, &resync_id).await;
if result.status == "failed" {
warn!(
event = EVENT_ADMIN_SITE_REPLICATION_STATE,
@@ -5798,60 +5657,10 @@ async fn refresh_bucket_targets_after_endpoint_edit(pending_id: &str, service_ac
Ok(())
}
async fn site_bucket_resync_manifest_entry(bucket: &str, peer: &PeerInfo, now: OffsetDateTime) -> ResyncBucketStatus {
let mut entry = ResyncBucketStatus {
bucket: bucket.to_string(),
status: "pending".to_string(),
created_at: Some(now),
updated_at: Some(now),
..Default::default()
};
let _targets_guard = lock_bucket_targets_metadata(bucket).await;
let (config, _) = match metadata_sys::get_replication_config(bucket).await {
Ok(config) => config,
Err(err) => {
entry.status = "failed".to_string();
entry.err_detail = summarize_peer_error_detail(&err.to_string());
return entry;
}
};
let targets = match metadata_sys::list_bucket_targets(bucket).await {
Ok(targets) => targets,
Err(err) => {
entry.status = "failed".to_string();
entry.err_detail = summarize_peer_error_detail(&err.to_string());
return entry;
}
};
let mut matching = targets
.targets
.iter()
.filter(|target| target.target_type == BucketTargetType::ReplicationService && bucket_target_matches_peer(target, peer));
let Some(target) = matching.next() else {
entry.status = "failed".to_string();
entry.err_detail = "no valid remote target found for peer".to_string();
return entry;
};
if matching.next().is_some() {
entry.status = "failed".to_string();
entry.err_detail = "multiple remote targets matched peer".to_string();
return entry;
}
let (has_arn, existing_object_enabled) = config.has_existing_object_replication(&target.arn);
if !has_arn || !existing_object_enabled {
entry.status = "failed".to_string();
entry.err_detail = "existing object replication is not enabled for the peer target".to_string();
return entry;
}
entry.target_arn = target.arn.clone();
entry
}
async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &str) -> ResyncBucketStatus {
async fn start_site_bucket_resync(bucket: &str, peer: &PeerInfo, resync_id: &str) -> ResyncBucketStatus {
let mut bucket_status = ResyncBucketStatus {
bucket: bucket.to_string(),
target_arn: target_arn.to_string(),
status: "running".to_string(),
status: "started".to_string(),
..Default::default()
};
let targets_guard = lock_bucket_targets_metadata(bucket).await;
@@ -5874,40 +5683,15 @@ async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &st
}
};
let Some(pool) = current_replication_pool_handle() else {
bucket_status.status = "failed".to_string();
bucket_status.err_detail = "replication pool is not initialized".to_string();
return bucket_status;
};
let Some(target_index) = targets
.targets
.iter()
.position(|target| target.target_type == BucketTargetType::ReplicationService && target.arn == target_arn)
else {
bucket_status.status = "failed".to_string();
bucket_status.err_detail = "recorded remote target no longer exists".to_string();
return bucket_status;
};
let existing_reset_id = targets.targets[target_index].reset_id.clone();
if !existing_reset_id.is_empty() && existing_reset_id != resync_id {
let existing_is_active = pool
.get_bucket_resync_status(bucket)
.await
.ok()
.and_then(|status| status.targets_map.get(target_arn).cloned())
.is_none_or(|target| target.resync_id != existing_reset_id || site_bucket_resync_is_active(target.resync_status));
if existing_is_active {
bucket_status.status = "conflict".to_string();
bucket_status.err_detail = "target belongs to a different active resync operation".to_string();
return bucket_status;
}
}
let reset_before = Some(OffsetDateTime::now_utc());
let target_arn = {
let target = &mut targets.targets[target_index];
let Some(target) = targets.targets.iter_mut().find(|target| {
target.target_type == BucketTargetType::ReplicationService && bucket_target_matches_peer(target, peer)
}) else {
bucket_status.status = "failed".to_string();
bucket_status.err_detail = format!("no valid remote target found for peer {}", peer.deployment_id);
return bucket_status;
};
let (has_arn, existing_object_enabled) = config.has_existing_object_replication(&target.arn);
if !has_arn || !existing_object_enabled {
@@ -5938,6 +5722,12 @@ async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &st
BucketTargetSys::get().update_all_targets(bucket, Some(&targets)).await;
drop(targets_guard);
let Some(pool) = current_replication_pool_handle() else {
bucket_status.status = "failed".to_string();
bucket_status.err_detail = "replication pool is not initialized".to_string();
return bucket_status;
};
if let Err(err) = pool
.start_bucket_resync(replication::resync_opts(bucket, target_arn, resync_id, reset_before))
.await
@@ -5949,10 +5739,9 @@ async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &st
bucket_status
}
async fn cancel_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &str) -> ResyncBucketStatus {
async fn cancel_site_bucket_resync(bucket: &str, peer: &PeerInfo, resync_id: &str) -> ResyncBucketStatus {
let mut bucket_status = ResyncBucketStatus {
bucket: bucket.to_string(),
target_arn: target_arn.to_string(),
status: "canceled".to_string(),
..Default::default()
};
@@ -5968,32 +5757,18 @@ async fn cancel_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &s
};
let Some(target) = targets.targets.iter_mut().find(|target| {
target.target_type == BucketTargetType::ReplicationService && target.arn == target_arn && target.reset_id == resync_id
target.target_type == BucketTargetType::ReplicationService
&& bucket_target_matches_peer(target, peer)
&& target.reset_id == resync_id
}) else {
bucket_status.status = "failed".to_string();
bucket_status.err_detail = "recorded resync target is not in progress".to_string();
bucket_status.err_detail = format!("no in-progress resync target found for peer {}", peer.deployment_id);
return bucket_status;
};
let target_arn = target.arn.clone();
let Some(pool) = current_replication_pool_handle() else {
bucket_status.status = "failed".to_string();
bucket_status.err_detail = "replication pool is not initialized".to_string();
return bucket_status;
};
if let Err(err) = pool
.cancel_bucket_resync(replication::resync_opts(bucket, target_arn, resync_id, None))
.await
{
bucket_status.status = "failed".to_string();
bucket_status.err_detail = err.to_string();
return bucket_status;
}
target.reset_id.clear();
target.reset_before_date = None;
let target_arn = target.arn.clone();
let json_targets = match serde_json::to_vec(&targets) {
Ok(json_targets) => json_targets,
@@ -6012,89 +5787,21 @@ async fn cancel_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &s
BucketTargetSys::get().update_all_targets(bucket, Some(&targets)).await;
drop(targets_guard);
bucket_status
}
let Some(pool) = current_replication_pool_handle() else {
bucket_status.status = "failed".to_string();
bucket_status.err_detail = "replication pool is not initialized".to_string();
return bucket_status;
};
async fn refresh_site_resync_status(mut status: SRResyncOpStatus, peer: &PeerInfo) -> SRResyncOpStatus {
for bucket in &mut status.buckets {
if bucket.target_arn.is_empty() && matches!(bucket.status.as_str(), "pending" | "running" | "started") {
let resolved = site_bucket_resync_manifest_entry(&bucket.bucket, peer, OffsetDateTime::now_utc()).await;
if resolved.target_arn.is_empty() {
bucket.status = "failed".to_string();
bucket.err_detail = resolved.err_detail;
} else {
bucket.target_arn = resolved.target_arn;
bucket.status = "pending".to_string();
}
}
}
if let Some(pool) = current_replication_pool_handle() {
for bucket in &mut status.buckets {
if bucket.target_arn.is_empty() || bucket.status == "failed" {
continue;
}
match pool.get_bucket_resync_status(&bucket.bucket).await {
Ok(live) => match live.targets_map.get(&bucket.target_arn) {
Some(target) if target.resync_id == status.resync_id => {
apply_site_resync_target_status(bucket, target);
}
Some(target) if !target.resync_id.is_empty() && site_bucket_resync_is_active(target.resync_status) => {
bucket.status = "conflict".to_string();
bucket.err_detail = "recorded target belongs to a different resync operation".to_string();
bucket.updated_at = Some(OffsetDateTime::now_utc());
}
Some(target) if !target.resync_id.is_empty() => {
bucket.status = "failed".to_string();
bucket.err_detail = "recorded resync operation was superseded by a terminal bucket resync".to_string();
bucket.updated_at = Some(OffsetDateTime::now_utc());
bucket.completed_at = bucket.updated_at;
}
_ if matches!(bucket.status.as_str(), "pending" | "running" | "started") => {
let previous = bucket.clone();
let mut recovered =
start_site_bucket_resync(&previous.bucket, &previous.target_arn, &status.resync_id).await;
recovered.created_at = previous.created_at;
recovered.started_at = previous.started_at.or(Some(OffsetDateTime::now_utc()));
recovered.updated_at = Some(OffsetDateTime::now_utc());
recovered.generation = status.generation;
recovered.err_detail = summarize_peer_error_detail(&recovered.err_detail);
*bucket = recovered;
}
_ => {}
},
Err(err) => {
bucket.err_detail = summarize_peer_error_detail(&err.to_string());
bucket.updated_at = Some(OffsetDateTime::now_utc());
}
}
}
}
summarize_site_resync_status(&mut status, OffsetDateTime::now_utc());
status
}
async fn persist_site_resync_status(peer_id: &str, status: &SRResyncOpStatus) -> S3Result<()> {
let _state_guard = SITE_REPLICATION_STATE_LOCK.lock().await;
let mut state = load_site_replication_state().await?;
if state
.resync_status
.get(peer_id)
.is_some_and(|current| current.resync_id != status.resync_id || current.generation != status.generation)
if let Err(err) = pool
.cancel_bucket_resync(replication::resync_opts(bucket, target_arn, resync_id, None))
.await
{
return Err(s3_error!(InvalidRequest, "site replication resync state changed"));
bucket_status.status = "failed".to_string();
bucket_status.err_detail = err.to_string();
}
state.resync_status.insert(peer_id.to_string(), status.clone());
save_site_replication_state(&state).await
}
async fn persist_new_site_resync_status(peer_id: &str, status: &SRResyncOpStatus) -> S3Result<()> {
let _state_guard = SITE_REPLICATION_STATE_LOCK.lock().await;
let mut state = load_site_replication_state().await?;
if state.resync_status.get(peer_id).is_some_and(site_resync_is_active) {
return Err(s3_error!(InvalidRequest, "site replication resync is already active"));
}
state.resync_status.insert(peer_id.to_string(), status.clone());
save_site_replication_state(&state).await
bucket_status
}
fn apply_state_edit_req(mut state: SiteReplicationState, body: SRStateEditReq) -> SiteReplicationState {
@@ -7554,162 +7261,89 @@ pub struct SiteReplicationResyncOpHandler {}
impl Operation for SiteReplicationResyncOpHandler {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
validate_site_replication_admin_request(&req, AdminAction::SiteReplicationResyncAction).await?;
let query = query_pairs(&req.uri);
let operation = query.get("operation").cloned().unwrap_or_default();
let operation = query_pairs(&req.uri).get("operation").cloned().unwrap_or_default();
// Resolve before the request body is consumed below; the `else` stays
// at its original position so error precedence is unchanged.
let resolved_store = object_store_from_req(&req);
let requested_peer: PeerInfo = read_site_replication_json(req, "", false).await?;
let _lifecycle_guard = SiteReplicationLifecycleGuard::acquire().await;
let (peer, existing_status) = {
let _state_guard = SITE_REPLICATION_STATE_LOCK.lock().await;
let state = load_site_replication_state().await?;
let local_peer = current_local_runtime_peer(&state);
let requested_peer = normalize_peer_info(requested_peer);
if requested_peer.deployment_id == local_peer.deployment_id {
return Err(s3_error!(InvalidRequest, "invalid peer specified - cannot resync to self"));
}
let peer = state
.peers
.get(&requested_peer.deployment_id)
.cloned()
.ok_or_else(|| s3_error!(InvalidRequest, "site replication peer not found"))?;
(peer, state.resync_status.get(&requested_peer.deployment_id).cloned())
let peer: PeerInfo = read_site_replication_json(req, "", false).await?;
let _state_guard = SITE_REPLICATION_STATE_LOCK.lock().await;
let mut state = load_site_replication_state().await?;
let local_peer = current_local_runtime_peer(&state);
let peer = normalize_peer_info(peer);
if peer.deployment_id == local_peer.deployment_id {
return Err(s3_error!(InvalidRequest, "invalid peer specified - cannot resync to self"));
}
if !state.peers.contains_key(&peer.deployment_id) {
return Err(s3_error!(InvalidRequest, "site replication peer not found"));
}
let Some(store) = resolved_store else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let buckets = store.list_bucket(&BucketOptions::default()).await.map_err(ApiError::from)?;
let bucket_names: Vec<String> = buckets.into_iter().map(|bucket| bucket.name).collect();
let mut status = match operation.as_str() {
let status = match operation.as_str() {
SITE_REPL_RESYNC_START => {
if let Some(existing) = existing_status.as_ref() {
let existing = refresh_site_resync_status(existing.clone(), &peer).await;
persist_site_resync_status(&peer.deployment_id, &existing).await?;
if site_resync_is_active(&existing) {
return Err(s3_error!(InvalidRequest, "site replication resync is already active"));
}
}
let Some(store) = resolved_store else {
return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string()));
};
let mut bucket_names: Vec<String> = store
.list_bucket(&BucketOptions::default())
.await
.map_err(ApiError::from)?
.into_iter()
.map(|bucket| bucket.name)
.collect();
bucket_names.sort();
let now = OffsetDateTime::now_utc();
let mut bucket_statuses = Vec::with_capacity(bucket_names.len());
let mut status = resync_status_for_state(&mut state, &operation, &peer, vec![]);
let mut bucket_statuses = Vec::new();
for bucket in bucket_names {
bucket_statuses.push(site_bucket_resync_manifest_entry(&bucket, &peer, now).await);
bucket_statuses.push(start_site_bucket_resync(&bucket, &peer, &status.resync_id).await);
}
let mut status = SRResyncOpStatus {
op_type: SITE_REPL_RESYNC_START.to_string(),
resync_id: Uuid::new_v4().to_string(),
status: "success".to_string(),
state: "pending".to_string(),
buckets: bucket_statuses,
created_at: Some(now),
started_at: Some(now),
updated_at: Some(now),
generation: existing_status
.as_ref()
.map_or(1, |existing| existing.generation.saturating_add(1).max(1)),
..Default::default()
};
summarize_site_resync_status(&mut status, now);
persist_new_site_resync_status(&peer.deployment_id, &status).await?;
for index in 0..status.buckets.len() {
if status.buckets[index].target_arn.is_empty() || status.buckets[index].status == "failed" {
continue;
}
let previous = status.buckets[index].clone();
let mut result = start_site_bucket_resync(&previous.bucket, &previous.target_arn, &status.resync_id).await;
result.created_at = previous.created_at;
result.started_at = Some(OffsetDateTime::now_utc());
result.updated_at = result.started_at;
result.generation = status.generation;
result.err_detail = summarize_peer_error_detail(&result.err_detail);
status.buckets[index] = result;
summarize_site_resync_status(&mut status, OffsetDateTime::now_utc());
persist_site_resync_status(&peer.deployment_id, &status).await?;
let failures = bucket_statuses.iter().filter(|bucket| bucket.status == "failed").count();
if failures == bucket_statuses.len() && !bucket_statuses.is_empty() {
status.status = "failed".to_string();
status.err_detail = "all buckets resync failed".to_string();
} else if failures > 0 {
status.err_detail = "partial failure in starting site resync".to_string();
}
status = refresh_site_resync_status(status, &peer).await;
persist_site_resync_status(&peer.deployment_id, &status).await?;
status.buckets = bucket_statuses;
state.resync_status.insert(peer.deployment_id.clone(), status.clone());
status
}
SITE_REPL_RESYNC_CANCEL => {
let Some(existing_status) = existing_status else {
let Some(existing_status) = state.resync_status.get(&peer.deployment_id).cloned() else {
return Err(s3_error!(InvalidRequest, "no resync in progress"));
};
if existing_status.resync_id.is_empty() {
return Err(s3_error!(InvalidRequest, "no resync in progress"));
}
let mut status = refresh_site_resync_status(existing_status, &peer).await;
if status.buckets.iter().any(|bucket| bucket.status == "conflict") {
return Err(s3_error!(
InvalidRequest,
"site replication resync target belongs to a different active operation"
));
let mut status = SRResyncOpStatus {
op_type: operation.clone(),
resync_id: existing_status.resync_id.clone(),
status: "success".to_string(),
..Default::default()
};
let mut bucket_statuses = Vec::new();
for bucket in bucket_names {
bucket_statuses.push(cancel_site_bucket_resync(&bucket, &peer, &existing_status.resync_id).await);
}
if site_resync_cancel_is_idempotent(&status) {
status.op_type = SITE_REPL_RESYNC_CANCEL.to_string();
for bucket in &status.buckets {
if !bucket.target_arn.is_empty() {
let _ = cancel_site_bucket_resync(&bucket.bucket, &bucket.target_arn, &status.resync_id).await;
}
}
status
} else {
if !site_resync_is_active(&status) {
return Err(s3_error!(InvalidRequest, "no active resync to cancel"));
}
status.op_type = SITE_REPL_RESYNC_CANCEL.to_string();
status.state = "canceling".to_string();
status.updated_at = Some(OffsetDateTime::now_utc());
persist_site_resync_status(&peer.deployment_id, &status).await?;
for index in 0..status.buckets.len() {
if status.buckets[index].target_arn.is_empty()
|| matches!(status.buckets[index].status.as_str(), "failed" | "canceled")
{
continue;
}
let previous = status.buckets[index].clone();
let mut result =
cancel_site_bucket_resync(&previous.bucket, &previous.target_arn, &status.resync_id).await;
result.created_at = previous.created_at;
result.started_at = previous.started_at;
result.updated_at = Some(OffsetDateTime::now_utc());
result.completed_at = result.updated_at;
result.generation = status.generation;
result.err_detail = summarize_peer_error_detail(&result.err_detail);
status.buckets[index] = result;
summarize_site_resync_status(&mut status, OffsetDateTime::now_utc());
persist_site_resync_status(&peer.deployment_id, &status).await?;
}
status = refresh_site_resync_status(status, &peer).await;
persist_site_resync_status(&peer.deployment_id, &status).await?;
status
let failures = bucket_statuses.iter().filter(|bucket| bucket.status == "failed").count();
if failures == bucket_statuses.len() && !bucket_statuses.is_empty() {
status.status = "failed".to_string();
status.err_detail = "all buckets resync cancel failed".to_string();
} else if failures > 0 {
status.err_detail = "partial failure in canceling site resync".to_string();
}
status.buckets = bucket_statuses;
state.resync_status.insert(peer.deployment_id.clone(), status.clone());
status
}
SITE_REPL_RESYNC_STATUS => {
let status = existing_status.unwrap_or_else(|| SRResyncOpStatus {
op_type: SITE_REPL_RESYNC_STATUS.to_string(),
status: "not-found".to_string(),
..Default::default()
});
if status.resync_id.is_empty() {
status
} else {
let status = refresh_site_resync_status(status, &peer).await;
persist_site_resync_status(&peer.deployment_id, &status).await?;
status
}
let status = state
.resync_status
.get(&peer.deployment_id)
.cloned()
.unwrap_or_else(|| SRResyncOpStatus {
op_type: SITE_REPL_RESYNC_STATUS.to_string(),
status: "not-found".to_string(),
..Default::default()
});
return json_response(&status);
}
_ => return Err(s3_error!(InvalidRequest, "unsupported resync operation")),
};
status
.buckets
.sort_by(|left, right| left.bucket.cmp(&right.bucket).then(left.target_arn.cmp(&right.target_arn)));
let (limit, offset) = parse_site_resync_page(&query, &status)?;
json_response(&site_resync_page(&status, limit, offset)?)
save_site_replication_state(&state).await?;
json_response(&status)
}
}
@@ -9226,57 +8860,6 @@ mod tests {
assert!(!policy.is_allowed(&put_policy_args).await);
}
// The replication service account must be able to carry object-lock metadata to the peer.
// Without these actions the peer answers AccessDenied for any replicated object that has
// retention or a legal hold, so a WORM-protected object never reaches the replica at all,
// and a retention change made after upload never propagates.
#[tokio::test]
async fn test_site_replicator_policy_allows_object_lock_replication() {
let policy = site_replicator_service_account_policy().expect("site replicator policy should parse");
let groups: Option<Vec<String>> = None;
let claims = HashMap::new();
let conditions = HashMap::new();
let base_args = rustfs_policy::policy::Args {
account: SITE_REPLICATOR_SERVICE_ACCOUNT,
groups: &groups,
action: Action::S3Action(S3Action::PutObjectRetentionAction),
conditions: &conditions,
is_owner: false,
claims: &claims,
deny_only: false,
bucket: "photos",
object: "image.jpg",
};
for action in [
S3Action::PutObjectRetentionAction,
S3Action::GetObjectRetentionAction,
S3Action::PutObjectLegalHoldAction,
S3Action::GetObjectLegalHoldAction,
] {
let args = rustfs_policy::policy::Args {
action: Action::S3Action(action),
..base_args
};
assert!(
policy.is_allowed(&args).await,
"site replicator must be allowed to replicate object-lock metadata: {action:?}"
);
}
// Governance bypass stays denied: replication must not be able to erase a retained
// version on the peer.
let bypass_args = rustfs_policy::policy::Args {
action: Action::S3Action(S3Action::BypassGovernanceRetentionAction),
..base_args
};
assert!(
!policy.is_allowed(&bypass_args).await,
"site replicator must not be granted governance bypass"
);
}
#[test]
fn test_sr_peer_edit_handler_uses_site_replication_operation_action() {
let src = include_str!("site_replication.rs");
@@ -11902,88 +11485,4 @@ mod tests {
assert!(rule_ids.contains(&"site-repl-dep-b"));
assert!(rule_ids.contains(&"site-repl-dep-c"));
}
#[test]
fn site_resync_summary_reports_partial_failure_and_clamps_counters() {
let now = OffsetDateTime::now_utc();
let mut running = ResyncBucketStatus {
bucket: "b".to_string(),
target_arn: "arn-b".to_string(),
..Default::default()
};
apply_site_resync_target_status(
&mut running,
&replication::TargetReplicationResyncStatus {
resync_status: replication::ResyncStatusType::ResyncStarted,
resync_id: "run-1".to_string(),
replicated_count: 4,
replicated_size: 16,
failed_count: -1,
failed_size: -2,
..Default::default()
},
);
let failed = ResyncBucketStatus {
bucket: "a".to_string(),
target_arn: "arn-a".to_string(),
status: "conflict".to_string(),
err_detail: "durable failure".to_string(),
..Default::default()
};
let mut status = SRResyncOpStatus {
op_type: SITE_REPL_RESYNC_START.to_string(),
resync_id: "run-1".to_string(),
buckets: vec![running, failed],
..Default::default()
};
summarize_site_resync_status(&mut status, now);
assert_eq!(status.status, "failed");
assert_eq!(status.state, "running");
assert_eq!(status.running_buckets, 1);
assert_eq!(status.failed_buckets, 1);
assert_eq!(status.replicated_objects, 4);
assert_eq!(status.replicated_bytes, 16);
assert_eq!(status.failed_objects, 0);
assert_eq!(status.failed_bytes, 0);
assert_eq!(status.completed_at, None);
assert!(site_resync_is_active(&status));
assert!(site_resync_cancel_is_idempotent(&SRResyncOpStatus {
state: "canceled".to_string(),
..Default::default()
}));
assert!(site_bucket_resync_is_active(replication::ResyncStatusType::ResyncPending));
assert!(site_bucket_resync_is_active(replication::ResyncStatusType::ResyncStarted));
assert!(!site_bucket_resync_is_active(replication::ResyncStatusType::ResyncCompleted));
}
#[test]
fn site_resync_pagination_is_sorted_and_rejects_stale_cursor() {
let status = SRResyncOpStatus {
resync_id: "run-1".to_string(),
generation: 3,
buckets: ["a", "b", "c"]
.into_iter()
.map(|bucket| ResyncBucketStatus {
bucket: bucket.to_string(),
..Default::default()
})
.collect(),
..Default::default()
};
let first = site_resync_page(&status, 2, 0).expect("first page should be valid");
assert!(first.truncated);
assert_eq!(first.buckets.iter().map(|bucket| bucket.bucket.as_str()).collect::<Vec<_>>(), ["a", "b"]);
let query = HashMap::from([
("limit".to_string(), "2".to_string()),
("continuationToken".to_string(), first.next_continuation_token),
]);
let (_, offset) = parse_site_resync_page(&query, &status).expect("cursor should match operation");
assert_eq!(offset, 2);
let mut newer = status;
newer.generation += 1;
assert!(parse_site_resync_page(&query, &newer).is_err());
}
}
+14 -165
View File
@@ -14,9 +14,6 @@
use super::{cluster_snapshot, metrics};
use crate::admin::auth::validate_admin_request;
use crate::admin::route_policy::{
ADMIN_ROUTE_POLICY_SPECS, DEFERRED_ADMIN_ROUTE_POLICIES, DeferredAdminRoutePolicy, DeferredRoutePolicyReason,
};
use crate::admin::router::{AdminOperation, Operation, S3Router};
use crate::admin::runtime_sources::{
DefaultAdminUsecase, QueryServerInfoRequest, current_endpoints_handle, default_admin_usecase, object_store_from_req,
@@ -34,10 +31,9 @@ use matchit::Params;
use rustfs_concurrency::WorkloadAdmissionRegistrySnapshot;
use rustfs_madmin::{InfoMessage, StorageInfo};
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
use rustfs_security_governance::{AdminRouteSpec, HttpMethod};
use s3s::header::CONTENT_TYPE;
use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error};
use serde::{Deserialize, Serialize};
use serde::Serialize;
use std::sync::atomic::{AtomicBool, Ordering};
use tracing::{error, info, warn};
@@ -58,9 +54,6 @@ const OBSERVABILITY_SUMMARY_RESOLVED: &str = "observability summary resolved fro
const TOPOLOGY_SUMMARY_RESOLVED: &str = "topology summary resolved from capability snapshot";
const TOPOLOGY_SNAPSHOT_NOT_AVAILABLE: &str = "endpoint topology is not available before storage endpoint pools initialize";
pub(crate) const RUNTIME_CAPABILITIES_ROUTE_SUFFIX: &str = "/v4/runtime/capabilities";
const SITE_REPLICATION_INFO_ROUTE: &str = "/rustfs/admin/v3/site-replication/info";
const SITE_REPLICATION_EDIT_ROUTE: &str = "/rustfs/admin/v3/site-replication/edit";
const SITE_REPLICATION_RESYNC_ROUTE: &str = "/rustfs/admin/v3/site-replication/resync/op";
macro_rules! log_system_request_rejected {
($operation:expr, $reason:expr) => {
@@ -616,7 +609,7 @@ impl Operation for StorageInfoHandler {
pub struct DataUsageInfoHandler {}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct RuntimeCapabilitiesSummary {
pub observability: CapabilityStatus,
pub userspace_profiling: CapabilityStatus,
@@ -624,12 +617,6 @@ pub struct RuntimeCapabilitiesSummary {
pub platform: CapabilityStatus,
pub topology: CapabilityStatus,
pub cluster_snapshot: CapabilityStatus,
#[serde(default)]
pub site_replication_info: CapabilityStatus,
#[serde(default)]
pub site_replication_edit: CapabilityStatus,
#[serde(default)]
pub site_replication_resync: CapabilityStatus,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
@@ -742,42 +729,9 @@ fn build_runtime_capabilities_summary(
cluster_snapshot: cluster_snapshot_summary.cloned().unwrap_or_else(|| {
CapabilityStatus::unknown().with_reason("cluster snapshot is not available before storage endpoint pools initialize")
}),
site_replication_info: admin_route_capability(HttpMethod::Get, SITE_REPLICATION_INFO_ROUTE),
site_replication_edit: admin_route_capability(HttpMethod::Put, SITE_REPLICATION_EDIT_ROUTE),
site_replication_resync: admin_route_capability(HttpMethod::Put, SITE_REPLICATION_RESYNC_ROUTE),
}
}
fn admin_route_capability(method: HttpMethod, path: &str) -> CapabilityStatus {
admin_route_capability_from_inventory(method, path, ADMIN_ROUTE_POLICY_SPECS, DEFERRED_ADMIN_ROUTE_POLICIES)
}
fn admin_route_capability_from_inventory(
method: HttpMethod,
path: &str,
direct: &[AdminRouteSpec],
deferred: &[DeferredAdminRoutePolicy],
) -> CapabilityStatus {
if direct.iter().any(|spec| spec.method() == method && spec.path() == path) {
return CapabilityStatus::supported().with_reason("public admin route is registered with an implemented handler");
}
match deferred
.iter()
.find(|policy| policy.method() == method && policy.path() == path)
.map(|policy| policy.reason())
{
Some(DeferredRoutePolicyReason::NotImplemented) | None => {
CapabilityStatus::unsupported().with_reason("public admin route is absent or not implemented")
}
Some(_) => CapabilityStatus::supported().with_reason("public admin route is registered with contextual authorization"),
}
}
fn runtime_capabilities_gate_actions() -> Vec<Action> {
vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)]
}
fn summarize_named_capability_statuses<const N: usize>(
statuses: [(&'static str, &CapabilityStatus); N],
subject: &'static str,
@@ -813,7 +767,15 @@ impl Operation for RuntimeCapabilitiesHandler {
check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?;
let remote_addr = req.extensions.get::<Option<RemoteAddr>>().and_then(|opt| opt.map(|a| a.0));
validate_admin_request(&req.headers, &cred, owner, false, runtime_capabilities_gate_actions(), remote_addr).await?;
validate_admin_request(
&req.headers,
&cred,
owner,
false,
vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)],
remote_addr,
)
.await?;
let response = build_runtime_capabilities_response().await.map_err(|err| {
log_system_request_failed!("runtime_capabilities", "build_runtime_capabilities_failed", err);
@@ -876,26 +838,18 @@ impl Operation for DataUsageInfoHandler {
#[cfg(test)]
mod tests {
use super::{
OBSERVABILITY_SUMMARY_RESOLVED, RuntimeCapabilitiesHandler, SITE_REPLICATION_EDIT_ROUTE, SITE_REPLICATION_INFO_ROUTE,
SITE_REPLICATION_RESYNC_ROUTE, ServerInfoResponse, TOPOLOGY_SNAPSHOT_NOT_AVAILABLE, TOPOLOGY_SUMMARY_RESOLVED,
admin_route_capability_from_inventory, build_runtime_capabilities_response, build_runtime_capabilities_summary,
data_usage_info_gate_actions, runtime_capabilities_gate_actions, system_admin_discovery,
OBSERVABILITY_SUMMARY_RESOLVED, ServerInfoResponse, TOPOLOGY_SNAPSHOT_NOT_AVAILABLE, TOPOLOGY_SUMMARY_RESOLVED,
build_runtime_capabilities_response, build_runtime_capabilities_summary, data_usage_info_gate_actions,
system_admin_discovery,
};
use crate::admin::router::Operation;
use crate::admin::runtime_sources::DefaultAdminUsecase;
use crate::admin::storage_api::cluster::{
CapabilityState, CapabilityStatus, MemorySamplingState, ObservabilitySnapshot, PlatformSupport, TopologyCapabilities,
TopologySnapshot, UserspaceProfilingCapability,
};
use http::{Extensions, HeaderMap, Uri};
use hyper::Method;
use matchit::Params;
use rustfs_concurrency::WorkloadClass;
use rustfs_madmin::{InfoMessage, StorageInfo};
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
use rustfs_security_governance::HttpMethod;
use s3s::{Body, S3ErrorCode, S3Request};
use serde_json::json;
/// Authz regression pin (rustfs/backlog#1306): datausageinfo stays an
/// any-of gate over exactly DataUsageInfoAdminAction OR ListBucketAction.
@@ -928,111 +882,6 @@ mod tests {
assert_eq!(response.summary.cluster_snapshot.state, CapabilityState::Unknown);
assert_eq!(response.observability.platform.os.as_deref(), Some(std::env::consts::OS));
assert_eq!(response.workload_admission.entries().len(), WorkloadClass::REQUIRED.len());
assert_eq!(response.summary.site_replication_info.state, CapabilityState::Supported);
assert_eq!(response.summary.site_replication_edit.state, CapabilityState::Supported);
assert_eq!(response.summary.site_replication_resync.state, CapabilityState::Supported);
let value = serde_json::to_value(response).expect("runtime capability response should serialize");
assert_eq!(value["summary"]["site_replication_info"]["state"], "supported");
assert_eq!(value["summary"]["site_replication_edit"]["state"], "supported");
assert_eq!(value["summary"]["site_replication_resync"]["state"], "supported");
}
#[test]
fn site_replication_capabilities_follow_public_route_inventory() {
for (method, path) in [
(HttpMethod::Get, SITE_REPLICATION_INFO_ROUTE),
(HttpMethod::Put, SITE_REPLICATION_EDIT_ROUTE),
(HttpMethod::Put, SITE_REPLICATION_RESYNC_ROUTE),
] {
assert_eq!(
admin_route_capability_from_inventory(
method,
path,
crate::admin::route_policy::ADMIN_ROUTE_POLICY_SPECS,
crate::admin::route_policy::DEFERRED_ADMIN_ROUTE_POLICIES,
)
.state,
CapabilityState::Supported,
);
}
}
#[test]
fn site_replication_capabilities_reject_absent_internal_only_and_not_implemented_routes() {
use crate::admin::route_policy::{DeferredAdminRoutePolicy, DeferredRoutePolicyReason};
let not_implemented = [DeferredAdminRoutePolicy::new(
HttpMethod::Put,
SITE_REPLICATION_EDIT_ROUTE,
DeferredRoutePolicyReason::NotImplemented,
)];
assert_eq!(
admin_route_capability_from_inventory(HttpMethod::Put, SITE_REPLICATION_EDIT_ROUTE, &[], &not_implemented).state,
CapabilityState::Unsupported,
);
assert_eq!(
admin_route_capability_from_inventory(HttpMethod::Put, SITE_REPLICATION_EDIT_ROUTE, &[], &[]).state,
CapabilityState::Unsupported,
);
let internal_only = crate::admin::route_policy::ADMIN_ROUTE_POLICY_SPECS
.iter()
.copied()
.filter(|spec| spec.path() == "/rustfs/admin/v3/site-replication/peer/edit-capabilities")
.collect::<Vec<_>>();
assert_eq!(
admin_route_capability_from_inventory(HttpMethod::Put, SITE_REPLICATION_EDIT_ROUTE, &internal_only, &[]).state,
CapabilityState::Unsupported,
);
}
#[test]
fn older_runtime_summary_defaults_site_replication_capabilities_to_unknown() {
let summary: super::RuntimeCapabilitiesSummary = serde_json::from_value(json!({
"observability": { "state": "supported" },
"userspace_profiling": { "state": "supported" },
"memory_sampling": { "state": "supported" },
"platform": { "state": "supported" },
"topology": { "state": "supported" },
"cluster_snapshot": { "state": "supported" }
}))
.expect("older runtime summary should remain compatible");
assert_eq!(summary.site_replication_info.state, CapabilityState::Unknown);
assert_eq!(summary.site_replication_edit.state, CapabilityState::Unknown);
assert_eq!(summary.site_replication_resync.state, CapabilityState::Unknown);
}
#[tokio::test]
async fn runtime_capabilities_authentication_failure_is_not_reported_as_unsupported() {
let request = S3Request {
input: Body::empty(),
method: Method::GET,
uri: Uri::from_static("/rustfs/admin/v4/runtime/capabilities"),
headers: HeaderMap::new(),
extensions: Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let error = RuntimeCapabilitiesHandler {}
.call(request, Params::new())
.await
.expect_err("runtime capabilities must reject unauthenticated requests before capability resolution");
assert_eq!(error.code(), &S3ErrorCode::InvalidRequest);
}
/// Authorization denial for this exact action is pinned to AccessDenied by
/// `crate::admin::auth::tests::non_admin_credential_is_denied`.
#[test]
fn runtime_capabilities_gate_requires_server_info_authorization() {
assert_eq!(
runtime_capabilities_gate_actions(),
vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)]
);
}
#[test]
+11 -302
View File
@@ -65,7 +65,7 @@ use rustfs_notify::{Event as NotificationEvent, notification_system};
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_s3_types::EventName;
use rustfs_signer::pre_sign_v4;
use rustfs_utils::egress::{OutboundDnsResolver, OutboundPolicy};
use rustfs_utils::egress::validate_outbound_url;
use rustfs_utils::http::{
SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_REPLICATION_CHECK, SUFFIX_SOURCE_REPLICATION_REQUEST,
SUFFIX_SOURCE_VERSION_ID, get_source_scheme, insert_header,
@@ -229,7 +229,7 @@ struct ListenNotificationFilter {
suffix: Option<String>,
}
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq, Eq)]
struct ObjectLambdaWebhookConfig {
endpoint: Url,
auth_token: String,
@@ -238,7 +238,6 @@ struct ObjectLambdaWebhookConfig {
client_ca: String,
skip_tls_verify: bool,
response_header_timeout: Option<Duration>,
outbound_resolver: OutboundDnsResolver,
}
const LAMBDA_WEBHOOK_SUB_SYS: &str = "lambda_webhook";
@@ -613,8 +612,7 @@ fn resolve_object_lambda_webhook_config_from_server_config(
let parsed_endpoint =
Url::parse(&endpoint).map_err(|_| s3_error!(InvalidRequest, "object lambda target endpoint is invalid"))?;
let outbound_resolver = outbound_policy()?
.resolver_for(&parsed_endpoint)
validate_outbound_url(&parsed_endpoint)
.map_err(|err| s3_error!(InvalidRequest, "object lambda target endpoint is not allowed: {}", err))?;
Ok(ObjectLambdaWebhookConfig {
@@ -625,7 +623,6 @@ fn resolve_object_lambda_webhook_config_from_server_config(
client_ca: kvs.lookup(WEBHOOK_CLIENT_CA).unwrap_or_default(),
skip_tls_verify: config_enable_is_on(&kvs.lookup(WEBHOOK_SKIP_TLS_VERIFY).unwrap_or_default()),
response_header_timeout,
outbound_resolver,
})
}
@@ -661,18 +658,9 @@ async fn resolve_object_lambda_webhook_config(uri: &Uri) -> S3Result<ObjectLambd
}
fn build_object_lambda_http_client(config: &ObjectLambdaWebhookConfig) -> S3Result<reqwest::Client> {
build_object_lambda_http_client_with_resolver(config, config.outbound_resolver.clone())
}
fn build_object_lambda_http_client_with_resolver(
config: &ObjectLambdaWebhookConfig,
resolver: impl reqwest::dns::Resolve + 'static,
) -> S3Result<reqwest::Client> {
let mut builder = reqwest::Client::builder()
.no_proxy()
.dns_resolver(resolver)
.redirect(reqwest::redirect::Policy::none())
.user_agent(rustfs_targets::get_user_agent(rustfs_targets::ServiceType::Basis));
validate_outbound_url(&config.endpoint)
.map_err(|err| s3_error!(InvalidRequest, "object lambda target endpoint is not allowed: {}", err))?;
let mut builder = reqwest::Client::builder().user_agent(rustfs_targets::get_user_agent(rustfs_targets::ServiceType::Basis));
if let Some(timeout) = config.response_header_timeout {
builder = builder.timeout(timeout);
@@ -684,7 +672,7 @@ fn build_object_lambda_http_client_with_resolver(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_OBJECT_LAMBDA,
result = "tls_verification_disabled",
endpoint_origin = %config.endpoint.origin().ascii_serialization(),
endpoint = %config.endpoint,
"admin router state"
);
builder = builder.danger_accept_invalid_certs(true);
@@ -718,17 +706,6 @@ fn build_object_lambda_http_client_with_resolver(
.map_err(|e| s3_error!(InternalError, "failed to build object lambda http client: {e}"))
}
async fn send_object_lambda_request(request: reqwest::RequestBuilder) -> S3Result<reqwest::Response> {
request
.send()
.await
.map_err(|_| s3_error!(InternalError, "object lambda target request failed"))
}
fn outbound_policy() -> S3Result<&'static OutboundPolicy> {
OutboundPolicy::from_env_cached().map_err(|err| s3_error!(InvalidRequest, "invalid outbound policy: {}", err))
}
fn extract_request_scheme(headers: &HeaderMap, uri: &Uri) -> String {
get_source_scheme(headers)
.and_then(|value| {
@@ -1121,7 +1098,10 @@ async fn invoke_object_lambda_target(
request_builder = request_builder.header("x-rustfs-object-lambda-version-id", version_id);
}
let lambda_response = send_object_lambda_request(request_builder).await?;
let lambda_response = request_builder
.send()
.await
.map_err(|e| s3_error!(InternalError, "object lambda target request failed: {e}"))?;
let status = lambda_response.status();
let lambda_headers = lambda_response.headers().clone();
@@ -2596,70 +2576,7 @@ mod tests {
use http::Method;
use http::Uri;
use s3s::S3Request;
use std::net::{IpAddr, SocketAddr};
use time::macros::datetime;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
#[derive(Clone)]
struct StaticResolver(IpAddr);
impl reqwest::dns::Resolve for StaticResolver {
fn resolve(&self, _name: reqwest::dns::Name) -> reqwest::dns::Resolving {
let address = SocketAddr::new(self.0, 0);
Box::pin(async move { Ok(Box::new(std::iter::once(address)) as reqwest::dns::Addrs) })
}
}
fn object_lambda_test_config(endpoint: Url) -> ObjectLambdaWebhookConfig {
object_lambda_test_config_with_policy(endpoint, &OutboundPolicy::default())
}
fn object_lambda_test_config_with_policy(endpoint: Url, policy: &OutboundPolicy) -> ObjectLambdaWebhookConfig {
let outbound_resolver = policy
.resolver_for(&endpoint)
.expect("test endpoint should satisfy its outbound policy");
ObjectLambdaWebhookConfig {
endpoint,
auth_token: String::new(),
client_cert: String::new(),
client_key: String::new(),
client_ca: String::new(),
skip_tls_verify: false,
response_header_timeout: Some(Duration::from_secs(2)),
outbound_resolver,
}
}
#[derive(Clone, Default)]
struct CapturedLog(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
struct CapturedLogWriter(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
impl std::io::Write for CapturedLogWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.lock().expect("captured log lock").extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'writer> tracing_subscriber::fmt::MakeWriter<'writer> for CapturedLog {
type Writer = CapturedLogWriter;
fn make_writer(&'writer self) -> Self::Writer {
CapturedLogWriter(self.0.clone())
}
}
impl CapturedLog {
fn contents(&self) -> String {
String::from_utf8(self.0.lock().expect("captured log lock").clone()).expect("captured logs must be UTF-8")
}
}
#[test]
fn canonicalize_admin_path_maps_compat_prefix_to_rustfs_prefix() {
@@ -3861,214 +3778,6 @@ mod tests {
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
}
#[test]
fn object_lambda_client_ignores_proxies_and_does_not_follow_redirects() {
const CHILD_ENV: &str = "RUSTFS_TEST_OBJECT_LAMBDA_PROXY_CHILD";
const TARGET_URL_ENV: &str = "RUSTFS_TEST_OBJECT_LAMBDA_TARGET_URL";
const TARGET_ADDR_ENV: &str = "RUSTFS_TEST_OBJECT_LAMBDA_TARGET_ADDR";
if std::env::var_os(CHILD_ENV).is_some() {
let endpoint = Url::parse(&std::env::var(TARGET_URL_ENV).expect("child target URL")).expect("target URL");
let address = std::env::var(TARGET_ADDR_ENV)
.expect("child target address")
.parse::<SocketAddr>()
.expect("target address");
let config = object_lambda_test_config(endpoint);
let client = build_object_lambda_http_client_with_resolver(&config, StaticResolver(address.ip()))
.expect("object lambda client should build");
tokio::runtime::Runtime::new().expect("child runtime").block_on(async {
let response = client
.get(config.endpoint)
.send()
.await
.expect("object lambda request should bypass environment proxy");
assert_eq!(response.status(), reqwest::StatusCode::FOUND);
});
return;
}
use std::io::{Read, Write};
use std::net::TcpListener as StdTcpListener;
let target_listener = StdTcpListener::bind("127.0.0.1:0").expect("bind object lambda target");
let target_address = target_listener.local_addr().expect("object lambda target address");
let redirect_listener = StdTcpListener::bind("127.0.0.1:0").expect("bind redirect destination");
let redirect_address = redirect_listener.local_addr().expect("redirect destination address");
drop(redirect_listener);
let target = std::thread::spawn(move || {
let (mut stream, _) = target_listener.accept().expect("accept object lambda request");
let mut request = [0_u8; 1024];
let read = stream.read(&mut request).expect("read object lambda request");
assert!(String::from_utf8_lossy(&request[..read]).starts_with("GET /transform HTTP/1.1"));
stream
.write_all(
format!(
"HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:{}/metadata\r\nContent-Length: 0\r\nConnection: close\r\n\r\n",
redirect_address.port()
)
.as_bytes(),
)
.expect("write object lambda redirect");
});
let proxy_listener = StdTcpListener::bind("127.0.0.1:0").expect("reserve refused proxy address");
let proxy_address = proxy_listener.local_addr().expect("proxy address");
drop(proxy_listener);
let target_url = format!("http://object-lambda.test:{}/transform", target_address.port());
let proxy_url = format!("http://{proxy_address}");
let output = std::process::Command::new(std::env::current_exe().expect("resolve current test executable"))
.arg("object_lambda_client_ignores_proxies_and_does_not_follow_redirects")
.arg("--nocapture")
.env(CHILD_ENV, "1")
.env(TARGET_URL_ENV, target_url)
.env(TARGET_ADDR_ENV, target_address.to_string())
.env("HTTP_PROXY", &proxy_url)
.env("HTTPS_PROXY", &proxy_url)
.env("ALL_PROXY", &proxy_url)
.env("NO_PROXY", "")
.output()
.expect("run isolated object lambda proxy test child");
assert!(
output.status.success(),
"proxy test child failed: stdout={} stderr={}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
target.join().expect("object lambda target should finish");
}
#[tokio::test]
async fn object_lambda_client_uses_configured_outbound_policy_resolver() {
let listener = TcpListener::bind("127.0.0.1:0")
.await
.expect("bind object lambda policy target");
let address = listener.local_addr().expect("object lambda policy target address");
let endpoint = Url::parse(&format!("http://{address}/transform")).expect("object lambda endpoint");
let policy = OutboundPolicy::from_allowed_origins(&endpoint.origin().ascii_serialization())
.expect("loopback test origin should be explicitly allowed");
let config = object_lambda_test_config_with_policy(endpoint.clone(), &policy);
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.expect("accept object lambda policy request");
let mut request = [0_u8; 1024];
let read = stream.read(&mut request).await.expect("read object lambda policy request");
assert!(String::from_utf8_lossy(&request[..read]).starts_with("GET /transform HTTP/1.1"));
stream
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.await
.expect("write object lambda policy response");
});
let response = build_object_lambda_http_client(&config)
.expect("object lambda policy client should build")
.get(endpoint)
.send()
.await
.expect("configured policy resolver should reach the allowed origin");
assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT);
server.await.expect("object lambda policy target should finish");
}
#[tokio::test]
async fn object_lambda_client_preserves_hostname_for_tls_sni() {
use rustls::{
ServerConfig, ServerConnection, StreamOwned,
pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer},
};
use std::io::{Read, Write};
use std::sync::{Arc, Once};
static INSTALL_CRYPTO_PROVIDER: Once = Once::new();
INSTALL_CRYPTO_PROVIDER.call_once(|| {
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
});
let rcgen::CertifiedKey { cert, signing_key } =
rcgen::generate_simple_self_signed(vec!["object-lambda.test".to_string()]).expect("cert should generate");
let temp_dir = tempfile::tempdir().expect("create temp directory");
let ca_path = temp_dir.path().join("object-lambda-ca.pem");
std::fs::write(&ca_path, cert.pem()).expect("write object lambda CA");
let server_config = Arc::new(
ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(
vec![cert.der().clone()],
PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(signing_key.serialize_der())),
)
.expect("server certificate should be valid"),
);
let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind object lambda TLS server");
let address = listener.local_addr().expect("object lambda TLS address");
let server = std::thread::spawn(move || {
let (stream, _) = listener.accept().expect("accept object lambda TLS request");
let connection = ServerConnection::new(server_config).expect("server TLS connection");
let mut stream = StreamOwned::new(connection, stream);
let mut request = [0_u8; 1024];
let _ = stream.read(&mut request).expect("read object lambda TLS request");
assert_eq!(stream.conn.server_name(), Some("object-lambda.test"));
stream
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.expect("write object lambda TLS response");
});
let endpoint =
Url::parse(&format!("https://object-lambda.test:{}/transform", address.port())).expect("object lambda TLS endpoint");
let mut config = object_lambda_test_config(endpoint.clone());
config.client_ca = ca_path.to_string_lossy().into_owned();
let response = build_object_lambda_http_client_with_resolver(&config, StaticResolver(address.ip()))
.expect("object lambda TLS client should build")
.get(endpoint)
.send()
.await
.expect("TLS request should keep the configured hostname for SNI");
assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT);
server.join().expect("object lambda TLS server should finish");
}
#[test]
fn object_lambda_tls_warning_redacts_endpoint_details() {
let captured = CapturedLog::default();
let subscriber = tracing_subscriber::fmt()
.with_ansi(false)
.without_time()
.with_max_level(tracing::Level::WARN)
.with_writer(captured.clone())
.finish();
let mut config = object_lambda_test_config(
Url::parse("https://object-lambda.test/private?token=secret").expect("object lambda endpoint"),
);
config.skip_tls_verify = true;
tracing::subscriber::with_default(subscriber, || {
build_object_lambda_http_client_with_resolver(&config, StaticResolver(IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)))
.expect("object lambda client should build");
});
let logs = captured.contents();
assert!(logs.contains("https://object-lambda.test"));
for secret in ["/private", "token=secret"] {
assert!(!logs.contains(secret), "TLS warning leaked {secret}: {logs}");
}
}
#[tokio::test]
async fn object_lambda_request_error_does_not_expose_endpoint_details() {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("reserve refused endpoint");
let address = listener.local_addr().expect("refused endpoint address");
drop(listener);
let request = reqwest::Client::builder()
.no_proxy()
.build()
.expect("test client should build")
.get(format!("http://{address}/private?token=secret"));
let error = send_object_lambda_request(request)
.await
.expect_err("refused object lambda request should fail");
let rendered = error.to_string();
assert!(rendered.contains("object lambda target request failed"));
assert!(!rendered.contains("/private"));
assert!(!rendered.contains("token=secret"));
}
#[test]
fn build_object_lambda_get_request_removes_lambda_arn_and_preserves_request_inputs() {
let mut req = S3Request {
+2
View File
@@ -323,7 +323,9 @@ pub(crate) mod replication {
pub(crate) type BucketStats = super::ecstore_bucket::replication::BucketStats;
pub(crate) type ReplicationStatusType = super::ecstore_bucket::replication::ReplicationStatusType;
pub(crate) type ResyncOpts = super::ecstore_bucket::replication::ResyncOpts;
#[cfg(test)]
pub(crate) type ResyncStatusType = super::ecstore_bucket::replication::ResyncStatusType;
#[cfg(test)]
pub(crate) type TargetReplicationResyncStatus = super::ecstore_bucket::replication::TargetReplicationResyncStatus;
pub(crate) fn resync_opts(
@@ -2089,78 +2089,3 @@ async fn put_object_computes_replication_decision_exactly_once() {
pre-commit + post-commit recompute makes this observe 2 (rustfs/backlog#1320)"
);
}
/// The object-lock handlers do not go through the object PUT path, so they must schedule
/// replication themselves. Without it a retention or legal hold applied after upload stays
/// local and the replica keeps its previous, unprotected state (a WORM object that is still
/// deletable on the peer). This observes the shared decision counter: each handler must
/// compute a replication decision exactly once, which is the step that drives both the
/// persisted pending marker and the schedule.
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state usecase integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn object_lock_handlers_schedule_replication() {
use super::storage_api::object_usecase::bucket::replication::MUST_REPLICATE_OBJECT_CALLS;
use s3s::S3;
use std::sync::atomic::Ordering;
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let bucket = format!("test-lock-repl-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object = "locked/object.txt";
(*ecstore)
.make_bucket(
bucket.as_str(),
&MakeBucketOptions {
lock_enabled: true,
versioning_enabled: true,
..Default::default()
},
)
.await
.expect("Failed to create object-lock bucket");
upload_test_object(&ecstore, bucket.as_str(), object, b"worm payload").await;
// PutObjectRetention must compute a replication decision.
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
let retention_input = PutObjectRetentionInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.retention(Some(ObjectLockRetention {
mode: Some(ObjectLockRetentionMode::from_static(ObjectLockRetentionMode::COMPLIANCE)),
retain_until_date: Some(time::macros::datetime!(2030-01-01 00:00:00 UTC).into()),
}))
.build()
.unwrap();
fs.put_object_retention(build_request(retention_input, Method::PUT))
.await
.expect("put_object_retention should succeed");
assert_eq!(
MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst),
1,
"PutObjectRetention must compute the replication decision exactly once; 0 means the \
retention change never replicates and the peer copy stays unprotected"
);
// PutObjectLegalHold must do the same.
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
let legal_hold_input = PutObjectLegalHoldInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.legal_hold(Some(ObjectLockLegalHold {
status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON)),
}))
.build()
.unwrap();
fs.put_object_legal_hold(build_request(legal_hold_input, Method::PUT))
.await
.expect("put_object_legal_hold should succeed");
assert_eq!(
MUST_REPLICATE_OBJECT_CALLS.load(Ordering::SeqCst),
1,
"PutObjectLegalHold must compute the replication decision exactly once; 0 means the \
legal hold never replicates and the peer copy stays deletable"
);
}
+250 -26
View File
@@ -197,8 +197,9 @@ use tracing::{debug, error, instrument, warn};
use uuid::Uuid;
use super::storage_api::object_usecase::{
GetObjectReader, StorageDeletedObject, StorageObjectInfo as ObjectInfo, StorageObjectLockDeleteOptions,
StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader,
GetObjectReader, StorageDeletedObject, StorageGetObjectSse as GetObjectSse, StorageObjectInfo as ObjectInfo,
StorageObjectLockDeleteOptions, StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete,
StoragePutObjReader as PutObjReader,
};
use crate::app::object_data_cache::{
ColdFillCoordinateOutcome, ColdFillDiskPermitOwner, ColdFillError, ColdFillProducer, GetObjectBodyCacheLookup,
@@ -3826,6 +3827,7 @@ impl DefaultObjectUsecase {
// metadata resolution.
let cache_hook_served = reader.is_cache_hook_served();
let cache_hook_probed = reader.cache_hook_probed();
let resolved_sse = reader.resolved_sse;
let info = reader.object_info;
let stream = reader.stream;
let buffered_body = reader.buffered_body;
@@ -3899,15 +3901,12 @@ impl DefaultObjectUsecase {
req.input.sse_customer_key.is_some()
);
let decryption_request = DecryptionRequest {
bucket,
key,
metadata: &info.user_defined,
sse_customer_key: req.input.sse_customer_key.as_ref(),
sse_customer_key_md5: req.input.sse_customer_key_md5.as_ref(),
};
let response_content_length = content_length;
// A cache-served body did not pass through ecstore's encryption-material
// resolver. Keep one KMS/SSE-C validation on that path; storage-backed
// readers have already resolved their material and only need response
// header classification here.
let resolved_sse = (!cache_hook_served).then_some(resolved_sse).flatten();
let (
server_side_encryption,
@@ -3917,22 +3916,60 @@ impl DefaultObjectUsecase {
encryption_applied,
final_stream,
buffered_body,
) = match sse_decryption(decryption_request).await? {
Some(material) => {
let server_side_encryption = Some(material.server_side_encryption.clone());
let sse_customer_algorithm = matches!(material.sse_type, SSEType::SseC).then_some(material.algorithm.clone());
let sse_customer_key_md5 = material.customer_key_md5.clone();
(
server_side_encryption,
sse_customer_algorithm,
sse_customer_key_md5,
material.kms_key_id,
true,
wrap_reader(stream),
None,
)
}
None => (None, None, None, None, false, wrap_reader(stream), buffered_body),
) = match resolved_sse {
Some(GetObjectSse::SseC { customer_key_md5 }) => (
Some(ServerSideEncryption::from_static(ServerSideEncryption::AES256)),
Some(SSECustomerAlgorithm::from(ServerSideEncryption::AES256.to_string())),
Some(customer_key_md5),
None,
true,
wrap_reader(stream),
None,
),
Some(GetObjectSse::SseS3) => (
Some(ServerSideEncryption::from_static(ServerSideEncryption::AES256)),
None,
None,
None,
true,
wrap_reader(stream),
None,
),
Some(GetObjectSse::SseKms { key_id }) => (
Some(ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS)),
None,
None,
Some(SSEKMSKeyId::from(key_id)),
true,
wrap_reader(stream),
None,
),
None if !cache_hook_served => (None, None, None, None, false, wrap_reader(stream), buffered_body),
None => match sse_decryption(DecryptionRequest {
bucket,
key,
metadata: &info.user_defined,
sse_customer_key: req.input.sse_customer_key.as_ref(),
sse_customer_key_md5: req.input.sse_customer_key_md5.as_ref(),
})
.await?
{
Some(material) => {
let server_side_encryption = Some(material.server_side_encryption.clone());
let sse_customer_algorithm = matches!(material.sse_type, SSEType::SseC).then_some(material.algorithm.clone());
let sse_customer_key_md5 = material.customer_key_md5.clone();
(
server_side_encryption,
sse_customer_algorithm,
sse_customer_key_md5,
material.kms_key_id,
true,
wrap_reader(stream),
None,
)
}
None => (None, None, None, None, false, wrap_reader(stream), buffered_body),
},
};
Ok(GetObjectReadSetup {
@@ -8096,6 +8133,185 @@ mod tests {
}
}
#[tokio::test]
async fn storage_resolved_sse_skips_second_app_layer_material_resolution() {
let input = GetObjectInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.build()
.expect("GET input must build");
let request = build_request(input, Method::GET);
let reader = GetObjectReader {
stream: Box::new(std::io::Cursor::new(b"body".to_vec())),
object_info: ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: 4,
actual_size: 4,
user_defined: Arc::new(HashMap::from([
("x-amz-server-side-encryption".to_string(), "AES256".to_string()),
("x-rustfs-encryption-key".to_string(), "not-base64".to_string()),
])),
..Default::default()
},
buffered_body: None,
resolved_sse: Some(GetObjectSse::SseS3),
body_source: GetObjectBodySource::Unprobed,
};
let setup = DefaultObjectUsecase::finish_get_object_read(
&request,
get_concurrency_manager(),
"bucket",
"object",
None,
None,
std::time::Instant::now(),
reader,
false,
)
.await
.expect("storage-resolved SSE must not reparse the malformed envelope in the app layer");
assert_eq!(
setup.server_side_encryption.as_ref().map(ServerSideEncryption::as_str),
Some(ServerSideEncryption::AES256)
);
assert!(setup.encryption_applied);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
#[serial_test::serial]
async fn cache_served_minio_static_kms_object_resolves_locally() {
use aes_gcm::{
Aes256Gcm, Nonce,
aead::{Aead, KeyInit, Payload},
};
use base64::{Engine, engine::general_purpose::STANDARD as BASE64_STANDARD};
use hmac::{Hmac, Mac};
use sha2::Sha256;
type HmacSha256 = Hmac<Sha256>;
let master_key = [0x31; 32];
let external_key = [0x42; 32];
let object_key = [0x53; 32];
let kms_iv = [0x64; 16];
let kms_nonce = [0x75; 12];
let mut kms_mac = HmacSha256::new_from_slice(&master_key).expect("valid KMS master key");
kms_mac.update(&kms_iv);
let kms_sealing_key = kms_mac.finalize().into_bytes();
let mut encrypted_dek = Aes256Gcm::new_from_slice(kms_sealing_key.as_slice())
.expect("valid KMS sealing key")
.encrypt(
&Nonce::from(kms_nonce),
Payload {
msg: &external_key,
aad: br#"{"bucket":"bucket/object"}"#,
},
)
.expect("encrypt static-KMS fixture DEK");
encrypted_dek.extend_from_slice(&kms_iv);
encrypted_dek.extend_from_slice(&kms_nonce);
let sealing_iv = [0x86; 32];
let mut object_mac = HmacSha256::new_from_slice(&external_key).expect("valid external key");
object_mac.update(&sealing_iv);
object_mac.update(b"SSE-KMS");
object_mac.update(b"DAREv2-HMAC-SHA256");
object_mac.update(b"bucket/object");
let object_sealing_key = object_mac.finalize().into_bytes();
let mut sealed_header = [0u8; 16];
sealed_header[0] = 0x20;
sealed_header[2..4].copy_from_slice(&(31u16).to_le_bytes());
sealed_header[4] = 0x80;
let mut sealed_object_key = sealed_header.to_vec();
sealed_object_key.extend_from_slice(
&Aes256Gcm::new_from_slice(object_sealing_key.as_slice())
.expect("valid object sealing key")
.encrypt(
&Nonce::from(<[u8; 12]>::try_from(&sealed_header[4..]).expect("12-byte nonce")),
Payload {
msg: &object_key,
aad: &sealed_header[..4],
},
)
.expect("seal object key"),
);
let input = GetObjectInput::builder()
.bucket("bucket".to_string())
.key("object".to_string())
.build()
.expect("GET input must build");
let request = build_request(input, Method::GET);
let metadata = HashMap::from([
("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()),
("x-amz-server-side-encryption-aws-kms-key-id".to_string(), "minio-key".to_string()),
(
"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id".to_string(),
"minio-key".to_string(),
),
(
"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key".to_string(),
BASE64_STANDARD.encode(encrypted_dek),
),
(
"X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key".to_string(),
BASE64_STANDARD.encode(sealed_object_key),
),
(
"X-Minio-Internal-Server-Side-Encryption-Iv".to_string(),
BASE64_STANDARD.encode(sealing_iv),
),
(
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm".to_string(),
"DAREv2-HMAC-SHA256".to_string(),
),
]);
let configured_key = format!("minio-key:{}", BASE64_STANDARD.encode(master_key));
temp_env::async_with_vars([("RUSTFS_MINIO_STATIC_KMS_KEY", Some(configured_key))], async {
let reader = GetObjectReader {
stream: Box::new(std::io::Cursor::new(b"body".to_vec())),
object_info: ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
size: 4,
actual_size: 4,
user_defined: Arc::new(metadata),
..Default::default()
},
buffered_body: Some(Bytes::from_static(b"body")),
resolved_sse: None,
body_source: GetObjectBodySource::HookServed,
};
let setup = DefaultObjectUsecase::finish_get_object_read(
&request,
get_concurrency_manager(),
"bucket",
"object",
None,
None,
std::time::Instant::now(),
reader,
false,
)
.await
.expect("cache-served MinIO static-KMS object must resolve without external KMS");
assert_eq!(
setup.server_side_encryption.as_ref().map(ServerSideEncryption::as_str),
Some(ServerSideEncryption::AWS_KMS)
);
assert_eq!(setup.ssekms_key_id.as_deref(), Some("minio-key"));
assert!(setup.encryption_applied);
})
.await;
}
#[test]
fn internal_object_info_lookup_opts_drops_http_preconditions() {
let version_id = Uuid::new_v4().to_string();
@@ -8911,6 +9127,7 @@ mod tests {
..Default::default()
},
buffered_body: None,
resolved_sse: None,
body_source: GetObjectBodySource::HookMissed,
})
},
@@ -9011,6 +9228,7 @@ mod tests {
..Default::default()
},
buffered_body: None,
resolved_sse: None,
body_source: GetObjectBodySource::HookMissed,
})
},
@@ -9614,6 +9832,7 @@ mod tests {
..Default::default()
},
buffered_body: Some(Bytes::from_static(b"body")),
resolved_sse: None,
body_source: GetObjectBodySource::HookMissed,
})
},
@@ -9788,6 +10007,7 @@ mod tests {
..Default::default()
},
buffered_body: Some(Bytes::from_static(b"body")),
resolved_sse: None,
body_source: GetObjectBodySource::HookMissed,
})
},
@@ -9918,6 +10138,7 @@ mod tests {
..Default::default()
},
buffered_body: None,
resolved_sse: None,
body_source: GetObjectBodySource::HookMissed,
})
},
@@ -9991,6 +10212,7 @@ mod tests {
..Default::default()
},
buffered_body: Some(Bytes::from_static(b"body")),
resolved_sse: None,
body_source: GetObjectBodySource::HookMissed,
})
},
@@ -10051,6 +10273,7 @@ mod tests {
..Default::default()
},
buffered_body: None,
resolved_sse: None,
body_source: GetObjectBodySource::HookMissed,
})
},
@@ -10182,6 +10405,7 @@ mod tests {
..Default::default()
},
buffered_body: None,
resolved_sse: None,
body_source: GetObjectBodySource::HookMissed,
})
},
+1 -1
View File
@@ -994,7 +994,7 @@ pub(crate) mod object_usecase {
object_utils, options, request_context, s3_api, set_disk, sse, storage_class, timeout_wrapper,
};
pub(crate) use crate::storage::storage_api::{
ECStore, GetObjectReader, OldCurrentSize, RFC1123, StorageDeletedObject, StorageObjectInfo,
ECStore, GetObjectReader, OldCurrentSize, RFC1123, StorageDeletedObject, StorageGetObjectSse, StorageObjectInfo,
StorageObjectLockDeleteOptions, StorageObjectOptions, StorageObjectToDelete, StoragePutObjReader, check_preconditions,
get_validated_store, has_replication_rules, parse_object_lock_legal_hold, parse_object_lock_retention,
parse_part_number_i32_to_usize, remove_object_lock_metadata_for_copy, strip_managed_encryption_metadata,
+4 -54
View File
@@ -262,27 +262,6 @@ fn secondary_tag_hint_action(action: Action, version_id: Option<&str>) -> Option
}
}
/// GHSA-3ppv: select the IAM action for an object read by whether the request
/// names an explicit object version. A read that targets a specific version must
/// authorize against `s3:GetObjectVersion`; a current-object read authorizes
/// against `s3:GetObject`. Reading a historical version while only holding
/// `s3:GetObject` is an information-disclosure bypass, so the two must not be
/// conflated at the authorization boundary.
///
/// Note: `ActionSet::is_match` still maps a `s3:GetObjectVersion` grant onto a
/// `s3:GetObject` request. That mapping is intentionally left in place until the
/// remaining version-aware read paths (HeadObject, GetObjectAcl, tagging) get the
/// same treatment — see the GHSA-3ppv follow-up audit. It does not re-open this
/// disclosure: it only broadens a Version grant toward current reads, never the
/// reverse.
fn versioned_read_action(version_id: Option<&str>) -> Action {
if version_id.is_some() {
Action::S3Action(S3Action::GetObjectVersionAction)
} else {
Action::S3Action(S3Action::GetObjectAction)
}
}
async fn get_or_fetch_object_tag_conditions<T>(
req: &mut S3Request<T>,
bucket: &str,
@@ -1046,9 +1025,7 @@ impl S3Access for FS {
req_info.object = Some(src_key.clone());
req_info.version_id = version_id.clone();
// GHSA-3ppv: a versioned copy source must authorize against
// s3:GetObjectVersion, not s3:GetObject.
authorize_request(req, versioned_read_action(version_id.as_deref())).await?;
authorize_request(req, Action::S3Action(S3Action::GetObjectAction)).await?;
}
let req_info = ext_req_info_mut(&mut req.extensions)?;
@@ -1493,9 +1470,7 @@ impl S3Access for FS {
req_info.object = Some(req.input.key.clone());
req_info.version_id = req.input.version_id.clone();
// GHSA-3ppv: a versioned read (?versionId=...) must authorize against
// s3:GetObjectVersion, not s3:GetObject.
authorize_request(req, versioned_read_action(req.input.version_id.as_deref())).await
authorize_request(req, Action::S3Action(S3Action::GetObjectAction)).await
}
/// Checks whether the GetObjectAcl request has accesses to the resources.
@@ -2075,9 +2050,7 @@ impl S3Access for FS {
req_info.object = Some(src_key.clone());
req_info.version_id = version_id.clone();
// GHSA-3ppv: a versioned copy source must authorize against
// s3:GetObjectVersion, not s3:GetObject.
authorize_request(req, versioned_read_action(version_id.as_deref())).await?;
authorize_request(req, Action::S3Action(S3Action::GetObjectAction)).await?;
}
let req_info = ext_req_info_mut(&mut req.extensions)?;
@@ -2107,7 +2080,7 @@ mod tests {
legal_hold_write_requested, list_parts_authorize_action, load_bucket_policy_existing_object_tag_hint,
merge_list_bucket_query_conditions, owner_can_bypass_policy_deny, post_object_authorize_action,
put_bucket_policy_authorize_action, request_context_from_req, retention_write_requested, secondary_tag_hint_action,
table_data_plane_admin_action, validate_post_object_success_controls, versioned_read_action,
table_data_plane_admin_action, validate_post_object_success_controls,
};
use crate::error::ApiError;
use http::{Extensions, HeaderMap, Method, Uri};
@@ -2140,29 +2113,6 @@ mod tests {
assert_eq!(get_bucket_policy_authorize_action(), Action::S3Action(S3Action::GetBucketPolicyAction));
}
#[test]
fn ghsa_3ppv_versioned_read_selects_get_object_version_action() {
// Regression (GHSA-3ppv): a read that names an explicit version must
// authorize against s3:GetObjectVersion so that a principal holding only
// s3:GetObject cannot read historical versions. Applies to GetObject and
// the CopyObject / UploadPartCopy sources.
assert_eq!(
versioned_read_action(Some("0194e0f1-0000-7000-8000-000000000000")),
Action::S3Action(S3Action::GetObjectVersionAction),
"an explicit versionId must require GetObjectVersion"
);
assert_eq!(
versioned_read_action(Some("null")),
Action::S3Action(S3Action::GetObjectVersionAction),
"the sentinel `null` version is still an explicit version selector"
);
assert_eq!(
versioned_read_action(None),
Action::S3Action(S3Action::GetObjectAction),
"a current-object read (no versionId) authorizes against GetObject"
);
}
#[test]
fn put_bucket_policy_uses_put_bucket_policy_action() {
assert_eq!(put_bucket_policy_authorize_action(), Action::S3Action(S3Action::PutBucketPolicyAction));
+9 -82
View File
@@ -45,7 +45,6 @@ use rustfs_targets::EventName;
use rustfs_utils::http::headers::{
AMZ_OBJECT_LOCK_LEGAL_HOLD_LOWER, AMZ_OBJECT_LOCK_MODE_LOWER, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE_LOWER,
};
use rustfs_utils::http::{SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TIMESTAMP, insert_str};
use s3s::{S3, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, dto::*, s3_error};
use std::fmt::Debug;
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
@@ -57,9 +56,6 @@ const LOG_SUBSYSTEM_OBJECT: &str = "object";
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
const LOG_SUBSYSTEM_TAGGING: &str = "tagging";
use crate::app::storage_api::object_usecase::bucket::replication::{
ReplicateDecision, must_replicate_object, schedule_object_replication,
};
use crate::storage::storage_api::ecfs_consumer::StorageObjectOptions as ObjectOptions;
#[derive(Debug, Clone)]
@@ -1286,53 +1282,20 @@ impl S3 for FS {
.await
.map_err(ApiError::from)?;
let mut popts = ObjectOptions {
let eval_metadata = parse_object_lock_legal_hold(legal_hold)?;
let popts = ObjectOptions {
mod_time: opts.mod_time,
version_id: opts.version_id.clone(),
version_id: opts.version_id,
eval_metadata: Some(eval_metadata),
..Default::default()
};
// PutObjectLegalHold only rewrites metadata, so replication is not scheduled by the
// object PUT path. Schedule it explicitly, otherwise the legal hold never reaches the
// replica and the peer copy remains deletable.
//
// Mirror the PUT path ordering (see object_usecase::put_object): compute the decision
// once BEFORE the commit and persist the pending marker with it, so a restart between
// the commit and the worker write-back leaves a Pending status on disk for the scanner
// to re-drive. Scheduling without that marker would lose the task silently.
let dsc = match store.get_object_info(&bucket, &key, &opts).await.ok() {
Some(info) => {
must_replicate_object(
&bucket,
&key,
&info.user_defined,
info.user_tags.as_ref().clone(),
popts.delete_marker_replication_status(),
popts.clone(),
)
.await
}
None => ReplicateDecision::new(),
};
let mut eval_metadata = parse_object_lock_legal_hold(legal_hold)?;
if dsc.replicate_any() {
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
}
popts.eval_metadata = Some(eval_metadata);
let info = store.put_object_metadata(&bucket, &key, &popts).await.map_err(|e| {
error!("put_object_metadata failed, {}", e.to_string());
s3_error!(InternalError, "{}", e.to_string())
})?;
// This replicates the whole object, not just the changed metadata: metadata-only
// replication is not implemented yet, so the lock change triggers a full re-upload.
if dsc.replicate_any() {
schedule_object_replication(info.clone(), store.clone(), dsc).await;
}
let output = PutObjectLegalHoldOutput {
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
};
@@ -1475,9 +1438,7 @@ impl S3 for FS {
.await
.map_err(ApiError::from)?;
let existing_obj_info = store.get_object_info(&bucket, &key, &check_opts).await.ok();
if let Some(existing_obj_info) = existing_obj_info.as_ref()
if let Ok(existing_obj_info) = store.get_object_info(&bucket, &key, &check_opts).await
&& let Some(block_reason) = check_retention_for_modification(
&existing_obj_info.user_defined,
new_mode.as_deref(),
@@ -1488,57 +1449,23 @@ impl S3 for FS {
return Err(S3Error::with_message(S3ErrorCode::AccessDenied, block_reason.error_message()));
}
let eval_metadata = parse_object_lock_retention(retention)?;
let mut opts: ObjectOptions = get_opts(&bucket, &key, version_id, None, &req.headers)
.await
.map_err(ApiError::from)?;
opts.eval_metadata = Some(eval_metadata);
opts.object_lock_retention = Some(ObjectLockRetentionOptions {
mode: new_mode,
retain_until: new_retain_until,
bypass_governance,
});
// PutObjectRetention only rewrites metadata, so the object PUT path that normally
// computes a replication decision and schedules replication never runs for it. Without
// scheduling here the peer keeps the previous, unprotected lock state and a
// WORM-protected object stays deletable on the replica.
//
// Mirror the PUT path ordering (see object_usecase::put_object): compute the decision
// once BEFORE the commit and persist the pending marker with it, so a restart between
// the commit and the worker write-back leaves a Pending status on disk for the scanner
// to re-drive. Scheduling without that marker would lose the task silently.
let dsc = match existing_obj_info.as_ref() {
Some(info) => {
must_replicate_object(
&bucket,
&key,
&info.user_defined,
info.user_tags.as_ref().clone(),
opts.delete_marker_replication_status(),
opts.clone(),
)
.await
}
None => ReplicateDecision::new(),
};
let mut eval_metadata = parse_object_lock_retention(retention)?;
if dsc.replicate_any() {
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_TIMESTAMP, jiff::Zoned::now().to_string());
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, dsc.pending_status().unwrap_or_default());
}
opts.eval_metadata = Some(eval_metadata);
let object_info = store.put_object_metadata(&bucket, &key, &opts).await.map_err(|e| {
error!("put_object_metadata failed, {}", e.to_string());
S3Error::from(ApiError::from(e))
})?;
// This replicates the whole object, not just the changed metadata: metadata-only
// replication is not implemented yet, so the lock change triggers a full re-upload.
if dsc.replicate_any() {
schedule_object_replication(object_info.clone(), store.clone(), dsc).await;
}
let output = PutObjectRetentionOutput {
request_charged: Some(RequestCharged::from_static(RequestCharged::REQUESTER)),
};
+1236 -505
View File
File diff suppressed because it is too large Load Diff
+11 -3
View File
@@ -82,6 +82,7 @@ pub(crate) mod contract {
pub(crate) type StorageDeletedObject = contract::object::DeletedObject;
pub(crate) type StorageGetObjectReader = super::GetObjectReader;
pub(crate) type StorageGetObjectSse = GetObjectSse;
pub(crate) type StorageObjectInfo = super::ObjectInfo;
pub(crate) type StorageObjectLockDeleteOptions = contract::object::ObjectLockDeleteOptions;
pub(crate) type StorageObjectOptions = super::ObjectOptions;
@@ -492,9 +493,9 @@ pub(crate) mod ecstore_object {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::object::GetObjectBodySource;
pub(crate) use rustfs_ecstore::api::object::{
GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, ObjectMutationHook, get_object_body_cache_plaintext_len,
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectSse, ObjectMutationHook,
get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
register_object_mutation_hook, unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
};
}
@@ -502,6 +503,12 @@ pub(crate) mod ecstore_set_disk {
pub(crate) use rustfs_ecstore::api::set_disk::{DEFAULT_READ_BUFFER_SIZE, get_lock_acquire_timeout, is_valid_storage_class};
}
pub(crate) mod ecstore_sse {
pub(crate) use rustfs_ecstore::api::sse::{ManagedDekProvider, ManagedSseScheme, classify_persisted_managed_encryption};
#[cfg(feature = "rio-v2")]
pub(crate) use rustfs_ecstore::api::sse::{PersistedManagedEncryption, decrypt_minio_static_kms_dek};
}
pub(crate) mod ecstore_storage {
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::storage::init_local_disks;
@@ -1559,6 +1566,7 @@ impl StorageVersioningConfigExt for s3s::dto::VersioningConfiguration {
}
pub(crate) type GetObjectReader = <ECStore as contract::object::ObjectIO>::GetObjectReader;
pub(crate) type GetObjectSse = ecstore_object::GetObjectSse;
pub(crate) type ObjectInfo = <ECStore as contract::object::ObjectOperations>::ObjectInfo;
pub(crate) type ObjectOptions = <ECStore as contract::object::ObjectOperations>::ObjectOptions;
pub(crate) type PutObjReader = <ECStore as contract::object::ObjectIO>::PutObjectReader;