mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-01 11:02:14 +00:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Zhengchao An
|
b7805caa58 | ||
|
Zhengchao An
|
b0bb0bbd3a | ||
|
GatewayJ
|
bc41e567a5 | ||
|
houseme
|
d5c6ba99d5 | ||
|
houseme
|
62d44d10b8 | ||
|
houseme
|
8218248000 | ||
|
Zhengchao An
|
fd36bdfb1a | ||
|
Zhengchao An
|
322ce21b9a |
@@ -300,9 +300,6 @@ path = "junit.xml"
|
||||
# negative-path siblings of each family stay in as regression guards.
|
||||
# * rustfs#4843 — over-limit archive entry paths hard-reject the whole
|
||||
# archive even under ignore-errors semantics.
|
||||
# * rustfs#4846 — distributed-lock quorum tests misclassify as timeout
|
||||
# under parallel load (multi-node in-process clusters; natural home is
|
||||
# ci-7's nightly cluster lane).
|
||||
[profile.e2e-full]
|
||||
default-filter = """
|
||||
package(e2e_test)
|
||||
@@ -311,7 +308,6 @@ default-filter = """
|
||||
& !test(/^replication_extension_test::/)
|
||||
& !test(/^multipart_auth_test::test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled$/)
|
||||
& !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(ignores_invalid_entries_when_requested|supports_standard_headers_with_combined_extract_options)$/)
|
||||
& !test(/^reliant::lock::test_distributed_lock_(2_nodes_grpc_read_survives_failed_node|4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes)$/)
|
||||
"""
|
||||
fail-fast = false
|
||||
|
||||
|
||||
@@ -111,7 +111,18 @@ runs:
|
||||
- name: Setup Rust cache
|
||||
uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
|
||||
with:
|
||||
cache-all-crates: true
|
||||
# false is rust-cache's own default. With true, cleanup.ts returns
|
||||
# *before* pruning ~/.cargo/registry/src, and config.ts archives the
|
||||
# whole registry — so every cache carried the unpacked source tree of
|
||||
# every dependency, not just "a few extra crates".
|
||||
#
|
||||
# No coverage is lost: getPackages runs `cargo metadata --all-features`,
|
||||
# a strict superset of any single lane's feature closure, and -sys crates
|
||||
# are explicitly exempted from pruning (their src timestamps would
|
||||
# otherwise trigger rebuilds). Anything pruned is re-unpacked from the
|
||||
# .crate files still in registry/cache, whose mtimes crates.io
|
||||
# normalises, so cargo fingerprints stay valid.
|
||||
cache-all-crates: false
|
||||
cache-on-failure: true
|
||||
shared-key: ${{ inputs.cache-shared-key }}
|
||||
save-if: ${{ inputs.cache-save-if }}
|
||||
|
||||
@@ -100,7 +100,9 @@ jobs:
|
||||
- name: Setup Rust cache
|
||||
uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
|
||||
with:
|
||||
cache-all-crates: true
|
||||
# Same reasoning as the setup composite: true archives every
|
||||
# dependency's unpacked source tree.
|
||||
cache-all-crates: false
|
||||
cache-on-failure: true
|
||||
shared-key: rustfs-cargo-deny
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
@@ -107,10 +107,6 @@ jobs:
|
||||
cache-save-if: 'true'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
# --all-targets covers the test binaries nextest builds, including
|
||||
# e2e_test, which test-and-lint's own run excludes. The second build adds
|
||||
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
|
||||
# and that no lint lane enables.
|
||||
# rustfs/backlog#1601 gate. sccache can only cache compilation units whose
|
||||
# --emit includes link, so it covers workspace rlibs and nothing else:
|
||||
# clippy is metadata-only, and the ~100 test binaries, the rustfs bin and
|
||||
@@ -138,6 +134,10 @@ jobs:
|
||||
retention-days: 30
|
||||
if-no-files-found: error
|
||||
|
||||
# --all-targets covers the test binaries nextest builds, including
|
||||
# e2e_test, which test-and-lint's own run excludes. The second build adds
|
||||
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
|
||||
# and that no lint lane enables.
|
||||
- name: Build ci-dev superset
|
||||
env:
|
||||
# Same limit ci.yml puts on its nextest step: this builds the same
|
||||
@@ -148,6 +148,21 @@ jobs:
|
||||
cargo build --workspace --all-targets
|
||||
cargo build -p rustfs --bins --features e2e-test-hooks
|
||||
|
||||
# Runs before rust-cache's post step, so these are the sizes it is about
|
||||
# to archive. Reported so the cache-all-crates decision stays evidence-led:
|
||||
# registry/src is what that flag prunes, registry/cache is what the pruned
|
||||
# sources are re-unpacked from. See rustfs/backlog#1600.
|
||||
- name: Report cache input sizes
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "### Cache input sizes (ci-dev)"
|
||||
echo '```'
|
||||
du -sh ~/.cargo/registry/src ~/.cargo/registry/cache ~/.cargo/registry/index \
|
||||
~/.cargo/git target 2>/dev/null || true
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2.
|
||||
warm-ci-feat-rio:
|
||||
name: Warm ci-feat-rio
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Asserts that the self-hosted runners are still ephemeral — one job per pod.
|
||||
#
|
||||
# This repository is public and its pull_request jobs run on those runners,
|
||||
# executing the PR's own build.rs, proc-macros and tests. The only thing keeping
|
||||
# that code from reaching a later job is that each ARC pod handles exactly one
|
||||
# job and is then destroyed. That guarantee lives in the ARC scale-set
|
||||
# configuration, outside this repository, where it can be changed without any PR
|
||||
# — so it is asserted here from the outside, against real run data, instead of
|
||||
# being assumed.
|
||||
#
|
||||
# Monthly rather than per-PR: the property changes only when someone
|
||||
# reconfigures the scale set, and the check costs a few dozen API calls.
|
||||
# See docs/ci/runners.md and rustfs/backlog#1602.
|
||||
|
||||
name: Runner Hygiene
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: runner-hygiene
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
check-ephemerality:
|
||||
name: Check runner ephemerality
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Exit 2 (inconclusive / broken) is deliberately not a pass: a window
|
||||
# where every sm-* job was still queued would otherwise look identical to
|
||||
# a clean bill of health.
|
||||
- name: Assert one job per self-hosted runner
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: ./scripts/ci/check_runner_ephemerality.sh 40
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [check-ephemerality]
|
||||
# Same ci-8 mechanism as coverage.yml, audit.yml and the nightly lanes:
|
||||
# scheduled runs file a tracking issue, manual dispatch stays quiet so
|
||||
# debugging never produces a spurious alert.
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
Generated
+149
-36
@@ -290,11 +290,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ar_archive_writer"
|
||||
version = "0.5.2"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348"
|
||||
checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6"
|
||||
dependencies = [
|
||||
"object 0.37.3",
|
||||
"object 0.39.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -599,6 +599,16 @@ dependencies = [
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "assert-json-diff"
|
||||
version = "2.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "astral-tokio-tar"
|
||||
version = "0.6.4"
|
||||
@@ -943,6 +953,32 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-kms"
|
||||
version = "1.114.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0b7d906608ee41e7ddea9983577ba82200435644d567d63dc34e822e088b453"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-observability",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.5.0",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.140.0"
|
||||
@@ -1158,17 +1194,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-protocol-test",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"h2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-util",
|
||||
"indexmap 2.14.0",
|
||||
"pin-project-lite",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
"rustls-pki-types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tower",
|
||||
@@ -1195,6 +1237,25 @@ dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-protocol-test"
|
||||
version = "0.64.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f76511a0e223ce78deb6a78b8afebda99cb737cfbc8a58d96dcb190f012dd40a"
|
||||
dependencies = [
|
||||
"assert-json-diff",
|
||||
"aws-smithy-runtime-api",
|
||||
"base64-simd",
|
||||
"cbor-diag",
|
||||
"ciborium",
|
||||
"http 0.2.12",
|
||||
"pretty_assertions",
|
||||
"regex-lite",
|
||||
"roxmltree",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-query"
|
||||
version = "0.62.0"
|
||||
@@ -1528,7 +1589,7 @@ version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1547,7 +1608,7 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1598,7 +1659,7 @@ version = "3.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f"
|
||||
dependencies = [
|
||||
"darling 0.20.11",
|
||||
"darling 0.23.0",
|
||||
"ident_case",
|
||||
"prettyplease",
|
||||
"proc-macro2",
|
||||
@@ -1679,9 +1740,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bytesize"
|
||||
version = "2.4.2"
|
||||
version = "2.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d7c8918969267b2932ffd5655509bbbea0833823058c378876953217f5fc50e"
|
||||
checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493"
|
||||
|
||||
[[package]]
|
||||
name = "bytestring"
|
||||
@@ -1758,6 +1819,25 @@ dependencies = [
|
||||
"cipher 0.5.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cbor-diag"
|
||||
version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429"
|
||||
dependencies = [
|
||||
"bs58",
|
||||
"chrono",
|
||||
"data-encoding",
|
||||
"half",
|
||||
"nom 7.1.3",
|
||||
"num-bigint",
|
||||
"num-rational",
|
||||
"num-traits",
|
||||
"separator",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.4.0"
|
||||
@@ -1870,7 +1950,7 @@ version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
@@ -1888,9 +1968,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.4"
|
||||
version = "4.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
|
||||
checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
@@ -1898,9 +1978,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.2"
|
||||
version = "4.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
|
||||
checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
@@ -2328,7 +2408,7 @@ version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -2353,11 +2433,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
@@ -3554,7 +3634,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -3814,7 +3894,7 @@ dependencies = [
|
||||
"crypto-bigint 0.5.5",
|
||||
"digest 0.10.7",
|
||||
"ff 0.13.1",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"group 0.13.0",
|
||||
"hkdf 0.12.4",
|
||||
"pem-rfc7468 0.7.0",
|
||||
@@ -4259,9 +4339,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -4274,7 +4354,7 @@ version = "1.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"rustversion",
|
||||
"typenum",
|
||||
]
|
||||
@@ -5301,7 +5381,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding 0.3.3",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5835,8 +5915,7 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
|
||||
[[package]]
|
||||
name = "libmimalloc-sys"
|
||||
version = "0.1.49"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=1cdadea43e9c5a0f054b65be21200ce580e4eb13#1cdadea43e9c5a0f054b65be21200ce580e4eb13"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cty",
|
||||
@@ -6245,8 +6324,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "mimalloc"
|
||||
version = "0.1.52"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=1cdadea43e9c5a0f054b65be21200ce580e4eb13#1cdadea43e9c5a0f054b65be21200ce580e4eb13"
|
||||
dependencies = [
|
||||
"libmimalloc-sys",
|
||||
]
|
||||
@@ -6668,6 +6746,17 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-rational"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
|
||||
dependencies = [
|
||||
"num-bigint",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
@@ -6732,7 +6821,7 @@ version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||
dependencies = [
|
||||
"base64 0.21.7",
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"getrandom 0.2.17",
|
||||
"http 1.5.0",
|
||||
@@ -7857,7 +7946,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"itertools 0.13.0",
|
||||
"itertools 0.14.0",
|
||||
"log",
|
||||
"multimap",
|
||||
"once_cell",
|
||||
@@ -7877,7 +7966,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"itertools 0.13.0",
|
||||
"itertools 0.14.0",
|
||||
"log",
|
||||
"multimap",
|
||||
"petgraph 0.8.3",
|
||||
@@ -7898,7 +7987,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.13.0",
|
||||
"itertools 0.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -7911,7 +8000,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.13.0",
|
||||
"itertools 0.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -8631,6 +8720,15 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "roxmltree"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b"
|
||||
dependencies = [
|
||||
"xmlparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rsa"
|
||||
version = "0.9.10"
|
||||
@@ -8724,9 +8822,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "russh"
|
||||
version = "0.62.4"
|
||||
version = "0.62.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8b67b5a0d8068c89dcbe9d95df986af7a851d1f3c604525274c37468e60464f"
|
||||
checksum = "da7c230e0ed9cbeb92fbad6c8848985d6df2a1464c0dc247a021abd666e9005e"
|
||||
dependencies = [
|
||||
"aes 0.9.2",
|
||||
"aws-lc-rs",
|
||||
@@ -9511,10 +9609,16 @@ dependencies = [
|
||||
"arc-swap",
|
||||
"argon2",
|
||||
"async-trait",
|
||||
"aws-config",
|
||||
"aws-sdk-kms",
|
||||
"aws-smithy-http-client",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"base64 0.23.0",
|
||||
"chacha20poly1305",
|
||||
"hex",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"insta",
|
||||
"jiff",
|
||||
"md-5 0.11.0",
|
||||
@@ -9709,6 +9813,7 @@ dependencies = [
|
||||
"hotpath",
|
||||
"jiff",
|
||||
"libc",
|
||||
"log",
|
||||
"metrics",
|
||||
"num_cpus",
|
||||
"nvml-wrapper",
|
||||
@@ -9744,6 +9849,7 @@ dependencies = [
|
||||
"tracing-error",
|
||||
"tracing-opentelemetry",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
@@ -9775,6 +9881,7 @@ dependencies = [
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10609,7 +10716,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
dependencies = [
|
||||
"base16ct 0.2.0",
|
||||
"der 0.7.10",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"pkcs8 0.10.2",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -10662,6 +10769,12 @@ dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "separator"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5"
|
||||
|
||||
[[package]]
|
||||
name = "seq-macro"
|
||||
version = "0.3.6"
|
||||
@@ -11599,7 +11712,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
|
||||
+6
-4
@@ -173,7 +173,7 @@ tower-http = { version = "0.7.0" }
|
||||
# Serialization and Data Formats
|
||||
apache-avro = "0.21.0"
|
||||
bytes = { version = "1.12.1" }
|
||||
bytesize = "2.4.2"
|
||||
bytesize = "2.6.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
form_urlencoded = "1.2.2"
|
||||
@@ -227,6 +227,7 @@ atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.10.1" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.114.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.140.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.2.0" }
|
||||
@@ -235,7 +236,7 @@ aws-smithy-types = { version = "1.6.1" }
|
||||
base64 = "0.23.0"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.4"
|
||||
clap = { version = "4.6.4" }
|
||||
clap = { version = "4.6.5" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.11.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -340,14 +341,15 @@ libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.4" }
|
||||
russh = { version = "0.62.5" }
|
||||
russh-sftp = "2.3.0"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = "0.1.52"
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13", features = ["extended"] }
|
||||
hotpath = { version = "0.22.0", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
@@ -126,10 +126,7 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
|
||||
assert_eq!(cancelled["success"], true);
|
||||
assert_eq!(cancelled["key_metadata"]["key_state"], "Enabled");
|
||||
|
||||
// A default server refuses to skip the waiting window (rustfs/backlog#1585):
|
||||
// immediate deletion is unrecoverable and takes every object encrypted under
|
||||
// the key with it, so the endpoint must reject it rather than honour it.
|
||||
let refused = kms_admin_request(
|
||||
let removed = kms_admin_request(
|
||||
base_url,
|
||||
http::Method::DELETE,
|
||||
"/rustfs/admin/v3/kms/keys/delete",
|
||||
@@ -143,49 +140,37 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
|
||||
access_key,
|
||||
secret_key,
|
||||
)
|
||||
.await
|
||||
.err()
|
||||
.ok_or("immediate KMS key deletion must be refused on a default server")?;
|
||||
assert!(
|
||||
refused.to_string().contains("400 Bad Request"),
|
||||
"refused immediate deletion must report a client error: {refused}"
|
||||
);
|
||||
|
||||
// The refused request left the key alone, so the window-bounded path still
|
||||
// has something to schedule.
|
||||
let described = kms_admin_request(
|
||||
base_url,
|
||||
http::Method::GET,
|
||||
&format!("/rustfs/admin/v3/kms/keys/{key_id}"),
|
||||
None,
|
||||
access_key,
|
||||
secret_key,
|
||||
)
|
||||
.await?;
|
||||
let described: serde_json::Value = serde_json::from_str(&described)?;
|
||||
assert_eq!(
|
||||
described["key_metadata"]["key_state"], "Enabled",
|
||||
"a refused immediate deletion must leave the key usable"
|
||||
);
|
||||
let removed: serde_json::Value = serde_json::from_str(&removed)?;
|
||||
assert_eq!(removed["success"], true);
|
||||
|
||||
let rescheduled = kms_admin_request(
|
||||
base_url,
|
||||
http::Method::DELETE,
|
||||
"/rustfs/admin/v3/kms/keys/delete",
|
||||
Some(
|
||||
&serde_json::json!({
|
||||
"key_id": key_id,
|
||||
"pending_window_in_days": 7
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
access_key,
|
||||
secret_key,
|
||||
)
|
||||
.await?;
|
||||
let rescheduled: serde_json::Value = serde_json::from_str(&rescheduled)?;
|
||||
assert_eq!(rescheduled["success"], true);
|
||||
assert!(rescheduled["deletion_date"].is_string());
|
||||
let listed =
|
||||
kms_admin_request(base_url, http::Method::GET, "/rustfs/admin/v3/kms/keys", None, access_key, secret_key).await?;
|
||||
let listed: serde_json::Value = serde_json::from_str(&listed)?;
|
||||
assert_eq!(listed["success"], true);
|
||||
let keys = listed["keys"]
|
||||
.as_array()
|
||||
.ok_or("list KMS keys response omitted keys after deletion")?;
|
||||
if let Some(key) = keys.iter().find(|key| key["key_id"] == key_id) {
|
||||
assert_eq!(key["status"], "PendingDeletion", "a retained force-deleted key must be pending deletion");
|
||||
let removed = kms_admin_request(
|
||||
base_url,
|
||||
http::Method::DELETE,
|
||||
"/rustfs/admin/v3/kms/keys/delete",
|
||||
Some(
|
||||
&serde_json::json!({
|
||||
"key_id": key_id,
|
||||
"force_immediate": true
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
access_key,
|
||||
secret_key,
|
||||
)
|
||||
.await?;
|
||||
let removed: serde_json::Value = serde_json::from_str(&removed)?;
|
||||
assert_eq!(removed["success"], true);
|
||||
}
|
||||
|
||||
let listed =
|
||||
kms_admin_request(base_url, http::Method::GET, "/rustfs/admin/v3/kms/keys", None, access_key, secret_key).await?;
|
||||
@@ -194,11 +179,10 @@ async fn assert_key_deletion_lifecycle(base_url: &str, access_key: &str, secret_
|
||||
let keys = listed["keys"]
|
||||
.as_array()
|
||||
.ok_or("final list KMS keys response omitted keys after deletion")?;
|
||||
let key = keys
|
||||
.iter()
|
||||
.find(|key| key["key_id"] == key_id)
|
||||
.ok_or("a key awaiting its deletion window must still be listed")?;
|
||||
assert_eq!(key["status"], "PendingDeletion", "a scheduled key must be pending deletion");
|
||||
assert!(
|
||||
keys.iter().all(|key| key["key_id"] != key_id),
|
||||
"force-deleted KMS key must no longer appear in list"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -449,32 +449,29 @@ async fn test_vault_kms_key_crud(
|
||||
|
||||
info!("✅ Delete verification: Key state correctly changed to: {}", key_state);
|
||||
|
||||
// Force Delete - a default server refuses to skip the waiting window
|
||||
// (rustfs/backlog#1585): destroying the key material immediately would take
|
||||
// every object encrypted under the key with it.
|
||||
let force_delete_error = crate::common::execute_awscurl(
|
||||
// Force Delete - Force immediate deletion for PendingDeletion key
|
||||
let force_delete_response = crate::common::execute_awscurl(
|
||||
&format!("{base_url}/rustfs/admin/v3/kms/keys/delete?keyId={key_id}&force_immediate=true"),
|
||||
"DELETE",
|
||||
None,
|
||||
access_key,
|
||||
secret_key,
|
||||
)
|
||||
.await
|
||||
.err()
|
||||
.expect("Immediate KMS key deletion must be refused on a default server");
|
||||
info!("✅ Force Delete: correctly refused for key {}: {}", key_id, force_delete_error);
|
||||
.await?;
|
||||
|
||||
// The refused request must leave the key exactly as it was: still present,
|
||||
// still pending deletion, still recoverable through cancel-deletion.
|
||||
let describe_after_refusal =
|
||||
crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await?;
|
||||
let describe_after_refusal: serde_json::Value = serde_json::from_str(&describe_after_refusal)?;
|
||||
assert_eq!(
|
||||
describe_after_refusal["key_metadata"]["key_state"], "PendingDeletion",
|
||||
"A refused immediate deletion must leave the key pending deletion"
|
||||
);
|
||||
// Parse and validate the force delete response
|
||||
let force_delete_result: serde_json::Value = serde_json::from_str(&force_delete_response)?;
|
||||
assert_eq!(force_delete_result["success"], true, "Force delete operation must return success=true");
|
||||
info!("✅ Force Delete: Successfully force deleted key: {}", key_id);
|
||||
|
||||
info!("✅ Force Delete verification: Key survived the refused immediate deletion");
|
||||
// Verify key no longer exists after force deletion (should return error)
|
||||
let describe_force_deleted_result =
|
||||
crate::common::awscurl_get(&format!("{base_url}/rustfs/admin/v3/kms/keys/{key_id}"), access_key, secret_key).await;
|
||||
|
||||
// After force deletion, key should not be found (GET should fail)
|
||||
assert!(describe_force_deleted_result.is_err(), "Force deleted key should not be found");
|
||||
|
||||
info!("✅ Force Delete verification: Key was permanently deleted and is no longer accessible");
|
||||
|
||||
info!("Vault KMS key CRUD operations completed successfully");
|
||||
Ok(())
|
||||
|
||||
@@ -15,9 +15,7 @@
|
||||
|
||||
use super::{grpc_lock_client::GrpcLockClient, grpc_lock_server::spawn_lock_server};
|
||||
use rustfs_lock::client::{LockClient, local::LocalClient};
|
||||
use rustfs_lock::{
|
||||
GlobalLockManager, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockType, NamespaceLock, ObjectKey,
|
||||
};
|
||||
use rustfs_lock::{GlobalLockManager, LockInfo, LockRequest, LockResponse, LockStats, LockType, NamespaceLock, ObjectKey};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -35,7 +33,11 @@ struct FailingClient;
|
||||
#[async_trait::async_trait]
|
||||
impl rustfs_lock::LockClient for FailingClient {
|
||||
async fn acquire_lock(&self, _request: &rustfs_lock::LockRequest) -> rustfs_lock::Result<LockResponse> {
|
||||
Err(LockError::internal("simulated gRPC node failure"))
|
||||
// Match RemoteClient's transport-failure response so the coordinator can count this node toward quorum loss.
|
||||
Ok(LockResponse::failure(
|
||||
"Remote lock RPC failed: simulated gRPC node failure",
|
||||
Duration::ZERO,
|
||||
))
|
||||
}
|
||||
|
||||
async fn release(&self, _lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
|
||||
|
||||
@@ -172,6 +172,9 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod replication {
|
||||
pub use crate::bucket::replication::replication_pool::{
|
||||
DurableMrfBacklogSummary, DurableMrfBucketBacklog, durable_mrf_backlog_summary_snapshot,
|
||||
};
|
||||
pub use crate::bucket::replication::{
|
||||
BucketReplicationResyncStatus, BucketStats, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool,
|
||||
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REPLICATE_INCOMING_DELETE, ReplicateDecision,
|
||||
|
||||
@@ -46,7 +46,11 @@ use super::replication_target_boundary::{ReplicationTargetStore, replication_obj
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str};
|
||||
use std::collections::HashMap;
|
||||
use std::collections::hash_map::Entry;
|
||||
use std::sync::Arc;
|
||||
use std::sync::LazyLock;
|
||||
use std::sync::RwLock as StdRwLock;
|
||||
use std::sync::atomic::AtomicI32;
|
||||
use std::sync::atomic::Ordering;
|
||||
use time::OffsetDateTime;
|
||||
@@ -74,6 +78,66 @@ pub struct DurableMrfBacklog {
|
||||
pub entries: Vec<MrfReplicateEntry>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct DurableMrfBucketBacklog {
|
||||
pub bucket: String,
|
||||
pub count: u64,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct DurableMrfBacklogSummary {
|
||||
pub available: bool,
|
||||
pub buckets: Vec<DurableMrfBucketBacklog>,
|
||||
}
|
||||
|
||||
static DURABLE_MRF_BACKLOG_SUMMARY: LazyLock<StdRwLock<DurableMrfBacklogSummary>> =
|
||||
LazyLock::new(|| StdRwLock::new(DurableMrfBacklogSummary::default()));
|
||||
|
||||
fn durable_mrf_backlog_summary_from_sizes<I>(entries: I) -> DurableMrfBacklogSummary
|
||||
where
|
||||
I: IntoIterator<Item = (String, i64)>,
|
||||
{
|
||||
let mut buckets = HashMap::<String, DurableMrfBucketBacklog>::new();
|
||||
for (bucket_name, entry_size) in entries {
|
||||
let Ok(size) = u64::try_from(entry_size) else {
|
||||
return DurableMrfBacklogSummary::default();
|
||||
};
|
||||
|
||||
let bucket = match buckets.entry(bucket_name) {
|
||||
Entry::Occupied(entry) => entry.into_mut(),
|
||||
Entry::Vacant(entry) => {
|
||||
let bucket = entry.key().clone();
|
||||
entry.insert(DurableMrfBucketBacklog {
|
||||
bucket,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
};
|
||||
bucket.count = bucket.count.saturating_add(1);
|
||||
bucket.bytes = bucket.bytes.saturating_add(size);
|
||||
}
|
||||
|
||||
DurableMrfBacklogSummary {
|
||||
available: true,
|
||||
buckets: buckets.into_values().collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn set_durable_mrf_backlog_summary(summary: DurableMrfBacklogSummary) {
|
||||
match DURABLE_MRF_BACKLOG_SUMMARY.write() {
|
||||
Ok(mut guard) => *guard = summary,
|
||||
Err(poisoned) => *poisoned.into_inner() = summary,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn durable_mrf_backlog_summary_snapshot() -> DurableMrfBacklogSummary {
|
||||
match DURABLE_MRF_BACKLOG_SUMMARY.read() {
|
||||
Ok(guard) => guard.clone(),
|
||||
Err(poisoned) => poisoned.into_inner().clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn durable_mrf_backlog_from_read(result: Result<Vec<u8>, EcstoreError>) -> DurableMrfBacklog {
|
||||
match result {
|
||||
Ok(data) => match decode_mrf_file(&data) {
|
||||
@@ -233,6 +297,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
let active_counter = self.active_lrg_workers.clone();
|
||||
let storage = self.storage.clone();
|
||||
let stats = self.stats.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let mut rx = rx;
|
||||
@@ -241,10 +306,18 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
let bucket = obj_info.bucket.clone();
|
||||
let size = obj_info.size;
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
let op_type = obj_info.op_type;
|
||||
replicate_object(*obj_info, storage.clone()).await;
|
||||
stats.dec_q(&bucket, size, delete_marker, op_type);
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
let bucket = del_info.bucket.clone();
|
||||
let op_type = del_info.op_type;
|
||||
replicate_delete(*del_info, storage.clone()).await;
|
||||
stats.dec_q(&bucket, 0, true, op_type);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,23 +383,22 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
stats
|
||||
.inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
|
||||
let bucket = obj_info.bucket.clone();
|
||||
let size = obj_info.size;
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
let op_type = obj_info.op_type;
|
||||
// Perform actual replication (placeholder)
|
||||
replicate_object(obj_info.as_ref().clone(), storage.clone()).await;
|
||||
replicate_object(*obj_info, storage.clone()).await;
|
||||
|
||||
stats
|
||||
.dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
stats.dec_q(&bucket, size, delete_marker, op_type);
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
stats.inc_q(&del_info.bucket, 0, true, del_info.op_type).await;
|
||||
let bucket = del_info.bucket.clone();
|
||||
let op_type = del_info.op_type;
|
||||
// Perform actual delete replication (placeholder)
|
||||
replicate_delete(del_info.as_ref().clone(), storage.clone()).await;
|
||||
replicate_delete(*del_info, storage.clone()).await;
|
||||
|
||||
stats.dec_q(&del_info.bucket, 0, true, del_info.op_type).await;
|
||||
stats.dec_q(&bucket, 0, true, op_type);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -379,16 +451,18 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
active_counter.fetch_add(1, Ordering::SeqCst);
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
stats
|
||||
.inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
replicate_object(obj_info.as_ref().clone(), storage.clone()).await;
|
||||
stats
|
||||
.dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
let bucket = obj_info.bucket.clone();
|
||||
let size = obj_info.size;
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
let op_type = obj_info.op_type;
|
||||
replicate_object(*obj_info, storage.clone()).await;
|
||||
stats.dec_q(&bucket, size, delete_marker, op_type);
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
let bucket = del_info.bucket.clone();
|
||||
let op_type = del_info.op_type;
|
||||
replicate_delete(*del_info, storage.clone()).await;
|
||||
stats.dec_q(&bucket, 0, true, op_type);
|
||||
}
|
||||
}
|
||||
active_counter.fetch_sub(1, Ordering::SeqCst);
|
||||
@@ -529,9 +603,13 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
if !lrg_workers.is_empty() {
|
||||
let index = (hash as usize) % lrg_workers.len();
|
||||
|
||||
if let Some(worker) = lrg_workers.get(index)
|
||||
&& worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err()
|
||||
{
|
||||
if let Some(worker) = lrg_workers.get(index) {
|
||||
self.stats.inc_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
|
||||
if worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_ok() {
|
||||
return ReplicationQueueAdmission::Queued;
|
||||
}
|
||||
self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
|
||||
|
||||
// Try to add more workers if possible
|
||||
let max_l_workers = *self.max_l_workers.read().await;
|
||||
let existing = lrg_workers.len();
|
||||
@@ -539,17 +617,13 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
drop(lrg_workers);
|
||||
|
||||
// Queue to MRF if worker is busy.
|
||||
let admission =
|
||||
queue_mrf_save_admission(&self.mrf_save_tx, ri.to_mrf_entry(), &ri.bucket, &ri.name, "large_object")
|
||||
.await;
|
||||
let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "large_object").await;
|
||||
|
||||
if let Some(resize) = resize {
|
||||
self.resize_lrg_workers(resize.new_count, resize.existing_count).await;
|
||||
}
|
||||
return admission;
|
||||
}
|
||||
|
||||
return ReplicationQueueAdmission::Queued;
|
||||
}
|
||||
return ReplicationQueueAdmission::Missed;
|
||||
}
|
||||
@@ -562,12 +636,14 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
return ReplicationQueueAdmission::Missed;
|
||||
};
|
||||
|
||||
self.stats.inc_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
|
||||
if channel.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_ok() {
|
||||
return ReplicationQueueAdmission::Queued;
|
||||
}
|
||||
self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
|
||||
|
||||
// Queue to MRF if all workers are busy.
|
||||
let admission = queue_mrf_save_admission(&self.mrf_save_tx, ri.to_mrf_entry(), &ri.bucket, &ri.name, "object").await;
|
||||
let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "object").await;
|
||||
|
||||
// Try to scale up workers based on priority
|
||||
self.apply_queue_backpressure("object", true, "Replication queue is backpressured")
|
||||
@@ -586,18 +662,13 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
return ReplicationQueueAdmission::Missed;
|
||||
};
|
||||
|
||||
self.stats.inc_q(&doi.bucket, 0, true, doi.op_type);
|
||||
if channel.try_send(ReplicationOperation::Delete(Box::new(doi.clone()))).is_ok() {
|
||||
return ReplicationQueueAdmission::Queued;
|
||||
}
|
||||
self.stats.dec_q(&doi.bucket, 0, true, doi.op_type);
|
||||
|
||||
let admission = queue_mrf_save_admission(
|
||||
&self.mrf_save_tx,
|
||||
doi.to_mrf_entry(),
|
||||
&doi.bucket,
|
||||
&doi.delete_object.object_name,
|
||||
"delete",
|
||||
)
|
||||
.await;
|
||||
let admission = self.queue_mrf_save_admission(doi.to_mrf_entry(), "delete").await;
|
||||
|
||||
self.apply_queue_backpressure("delete", false, "Replication delete queue is backpressured")
|
||||
.await;
|
||||
@@ -607,7 +678,18 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Queues an MRF save operation
|
||||
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
||||
let _ = queue_mrf_save_admission(&self.mrf_save_tx, entry, "", "", "mrf_worker").await;
|
||||
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
|
||||
}
|
||||
|
||||
async fn queue_mrf_save_admission(&self, entry: MrfReplicateEntry, queue_type: &'static str) -> ReplicationQueueAdmission {
|
||||
let bucket = entry.bucket.clone();
|
||||
let size = entry.size;
|
||||
let is_delete = matches!(entry.op, MrfOpKind::Delete);
|
||||
let admission = queue_mrf_save_entry(&self.mrf_save_tx, entry, queue_type).await;
|
||||
if admission == ReplicationQueueAdmission::Queued {
|
||||
self.stats.inc_q(&bucket, size, is_delete, ReplicationType::Heal);
|
||||
}
|
||||
admission
|
||||
}
|
||||
|
||||
/// Starts the MRF processor — one-shot at startup.
|
||||
@@ -622,7 +704,13 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let handle = tokio::spawn(async move {
|
||||
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
||||
Ok(d) => d,
|
||||
Err(EcstoreError::ConfigNotFound) => return, // no file yet — normal on first start
|
||||
Err(EcstoreError::ConfigNotFound) => {
|
||||
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
||||
available: true,
|
||||
buckets: Vec::new(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -653,6 +741,9 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
return;
|
||||
}
|
||||
};
|
||||
set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
|
||||
entries.iter().map(|entry| (entry.bucket.clone(), entry.size)),
|
||||
));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
@@ -765,6 +856,11 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
error = %e,
|
||||
"Failed to clear MRF recovery file after replay — entries may be replayed again on next restart"
|
||||
);
|
||||
} else {
|
||||
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
||||
available: true,
|
||||
buckets: Vec::new(),
|
||||
});
|
||||
}
|
||||
|
||||
if queued_count > 0 {
|
||||
@@ -793,6 +889,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
return;
|
||||
};
|
||||
let storage = self.storage.clone();
|
||||
let stats = self.stats.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
// The on-disk MRF file is a restart-recovery backstop: entries are
|
||||
@@ -818,6 +915,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
entry = rx.recv() => match entry {
|
||||
Some(e) => {
|
||||
if pending.len() >= MRF_PENDING_CAP {
|
||||
dec_mrf_entries(stats.as_ref(), std::slice::from_ref(&e));
|
||||
if !capped {
|
||||
capped = true;
|
||||
warn!(
|
||||
@@ -836,20 +934,31 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
// set, not the absolute length, so a large backlog is
|
||||
// not rewritten on every single add).
|
||||
if pending.len() - flushed_len >= 1000 && flush_mrf_to_disk(&pending, &storage).await {
|
||||
set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
|
||||
pending.iter().map(|entry| (entry.bucket.clone(), entry.size)),
|
||||
));
|
||||
dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
|
||||
flushed_len = pending.len();
|
||||
dirty = false;
|
||||
}
|
||||
}
|
||||
None => {
|
||||
// Channel closed (pool shutting down) — final flush.
|
||||
if dirty {
|
||||
flush_mrf_to_disk(&pending, &storage).await;
|
||||
if dirty && flush_mrf_to_disk(&pending, &storage).await {
|
||||
set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
|
||||
pending.iter().map(|entry| (entry.bucket.clone(), entry.size)),
|
||||
));
|
||||
dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
},
|
||||
_ = interval.tick() => {
|
||||
if dirty && flush_mrf_to_disk(&pending, &storage).await {
|
||||
set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
|
||||
pending.iter().map(|entry| (entry.bucket.clone(), entry.size)),
|
||||
));
|
||||
dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
|
||||
flushed_len = pending.len();
|
||||
dirty = false;
|
||||
}
|
||||
@@ -872,24 +981,22 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
stats
|
||||
.inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
|
||||
let bucket = obj_info.bucket.clone();
|
||||
let size = obj_info.size;
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
let op_type = obj_info.op_type;
|
||||
// Perform actual replication (placeholder)
|
||||
replicate_object(obj_info.as_ref().clone(), self.storage.clone()).await;
|
||||
replicate_object(*obj_info, self.storage.clone()).await;
|
||||
|
||||
stats
|
||||
.dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
stats.dec_q(&bucket, size, delete_marker, op_type);
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
stats.inc_q(&del_info.bucket, 0, true, del_info.op_type).await;
|
||||
|
||||
let bucket = del_info.bucket.clone();
|
||||
let op_type = del_info.op_type;
|
||||
// Perform actual delete replication (placeholder)
|
||||
replicate_delete(del_info.as_ref().clone(), self.storage.clone()).await;
|
||||
replicate_delete(*del_info, self.storage.clone()).await;
|
||||
|
||||
stats.dec_q(&del_info.bucket, 0, true, del_info.op_type).await;
|
||||
stats.dec_q(&bucket, 0, true, op_type);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -898,16 +1005,30 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling large object replication operations
|
||||
async fn add_large_worker(&self, mut rx: Receiver<ReplicationOperation>, active_counter: Arc<AtomicI32>, storage: Arc<S>) {
|
||||
async fn add_large_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
active_counter: Arc<AtomicI32>,
|
||||
stats: Arc<ReplicationStats>,
|
||||
storage: Arc<S>,
|
||||
) {
|
||||
while let Some(operation) = rx.recv().await {
|
||||
active_counter.fetch_add(1, Ordering::SeqCst);
|
||||
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
let bucket = obj_info.bucket.clone();
|
||||
let size = obj_info.size;
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
let op_type = obj_info.op_type;
|
||||
replicate_object(*obj_info, storage.clone()).await;
|
||||
stats.dec_q(&bucket, size, delete_marker, op_type);
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
let bucket = del_info.bucket.clone();
|
||||
let op_type = del_info.op_type;
|
||||
replicate_delete(*del_info, storage.clone()).await;
|
||||
stats.dec_q(&bucket, 0, true, op_type);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -927,18 +1048,19 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
stats
|
||||
.inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
|
||||
let bucket = obj_info.bucket.clone();
|
||||
let size = obj_info.size;
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
let op_type = obj_info.op_type;
|
||||
replicate_object(obj_info.as_ref().clone(), self.storage.clone()).await;
|
||||
|
||||
stats
|
||||
.dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
stats.dec_q(&bucket, size, delete_marker, op_type);
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
let bucket = del_info.bucket.clone();
|
||||
let op_type = del_info.op_type;
|
||||
replicate_delete(*del_info, self.storage.clone()).await;
|
||||
stats.dec_q(&bucket, 0, true, op_type);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1273,29 +1395,34 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn queue_mrf_save_admission(
|
||||
async fn queue_mrf_save_entry(
|
||||
tx: &Sender<MrfReplicateEntry>,
|
||||
entry: MrfReplicateEntry,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
queue_type: &'static str,
|
||||
) -> ReplicationQueueAdmission {
|
||||
if tx.send(entry).await.is_ok() {
|
||||
let Err(error) = tx.send(entry).await else {
|
||||
return ReplicationQueueAdmission::Queued;
|
||||
}
|
||||
};
|
||||
let entry = error.0;
|
||||
|
||||
warn!(
|
||||
event = EVENT_REPLICATION_MRF_QUEUE_UNAVAILABLE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
queue_type = queue_type,
|
||||
"MRF save channel unavailable — replication failure entry could not be persisted for retry"
|
||||
);
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
|
||||
fn dec_mrf_entries(stats: &ReplicationStats, entries: &[MrfReplicateEntry]) {
|
||||
for entry in entries {
|
||||
stats.dec_q(&entry.bucket, entry.size, matches!(entry.op, MrfOpKind::Delete), ReplicationType::Heal);
|
||||
}
|
||||
}
|
||||
|
||||
/// Encodes `entries` and overwrites the MRF persistence file.
|
||||
/// Returns `true` on success; on failure logs the error and returns `false`.
|
||||
/// Callers must NOT clear their in-memory buffer on `false` so the next tick
|
||||
@@ -2016,6 +2143,147 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
async fn current_queue(pool: &ReplicationPool<LoadResyncNodeStore>, bucket: &str) -> (i64, i64) {
|
||||
let stats = pool.stats.get_latest_replication_stats(bucket).await;
|
||||
(stats.replication_stats.q_stat.curr.count, stats.replication_stats.q_stat.curr.bytes)
|
||||
}
|
||||
|
||||
async fn wait_for_current_queue(pool: &ReplicationPool<LoadResyncNodeStore>, bucket: &str, expected: (i64, i64)) {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
if current_queue(pool, bucket).await == expected {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("replication queue should reach the expected state");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn regular_worker_admission_counts_channel_backlog_before_receive() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, _rx) = mpsc::channel(1);
|
||||
pool.workers.write().await.push(tx);
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_task(ReplicateObjectInfo {
|
||||
bucket: "admission-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 4096,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(current_queue(&pool, "admission-bucket").await, (1, 4096));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn large_worker_admission_counts_channel_backlog_before_receive() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, _rx) = mpsc::channel(1);
|
||||
pool.lrg_workers.write().await.push(tx);
|
||||
let size = 128 * 1024 * 1024;
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_task(ReplicateObjectInfo {
|
||||
bucket: "large-admission-bucket".to_string(),
|
||||
name: "large-object".to_string(),
|
||||
size,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(current_queue(&pool, "large-admission-bucket").await, (1, size));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_admission_counts_channel_backlog_before_receive() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, _rx) = mpsc::channel(1);
|
||||
pool.workers.write().await.push(tx);
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_delete_task(DeletedObjectReplicationInfo {
|
||||
bucket: "delete-admission-bucket".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "deleted-object".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(current_queue(&pool, "delete-admission-bucket").await, (1, 0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn regular_worker_drains_current_backlog_after_processing() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
pool.resize_workers(1, 0).await;
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_task(ReplicateObjectInfo {
|
||||
bucket: "regular-drain-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 4096,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
wait_for_current_queue(&pool, "regular-drain-bucket", (0, 0)).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn large_worker_drains_current_backlog_after_processing() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
pool.resize_lrg_workers(1, 0).await;
|
||||
let size = 128 * 1024 * 1024;
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_task(ReplicateObjectInfo {
|
||||
bucket: "large-drain-bucket".to_string(),
|
||||
name: "large-object".to_string(),
|
||||
size,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
wait_for_current_queue(&pool, "large-drain-bucket", (0, 0)).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn regular_delete_worker_drains_current_backlog_after_processing() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
pool.resize_workers(1, 0).await;
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_delete_task(DeletedObjectReplicationInfo {
|
||||
bucket: "delete-drain-bucket".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "deleted-object".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
wait_for_current_queue(&pool, "delete-drain-bucket", (0, 0)).await;
|
||||
}
|
||||
|
||||
fn load_resync_test_metadata() -> Vec<u8> {
|
||||
let mut status = BucketReplicationResyncStatus::new();
|
||||
status.targets_map.insert(
|
||||
@@ -2301,6 +2569,37 @@ mod tests {
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Missed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_counts_mrf_pending_backlog_when_worker_queue_is_full() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", shared))).await;
|
||||
let (tx, _rx) = mpsc::channel(1);
|
||||
tx.try_send(ReplicationOperation::Object(Box::new(ReplicateObjectInfo {
|
||||
bucket: "runtime-backlog".to_string(),
|
||||
name: "already-buffered".to_string(),
|
||||
size: 1,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})))
|
||||
.expect("test setup should fill the worker queue");
|
||||
pool.workers.write().await.push(tx);
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_task(ReplicateObjectInfo {
|
||||
bucket: "runtime-backlog".to_string(),
|
||||
name: "fallback-object".to_string(),
|
||||
size: 2048,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
let queued = pool.stats.get_latest_replication_stats("runtime-backlog").await;
|
||||
assert_eq!(queued.replication_stats.q_stat.curr.count, 1);
|
||||
assert_eq!(queued.replication_stats.q_stat.curr.bytes, 2048);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replicate_object_info_from_object_info_preserves_ssec_checksum() {
|
||||
let checksum = bytes::Bytes::from_static(b"ssec-checksum");
|
||||
@@ -2342,7 +2641,7 @@ mod tests {
|
||||
|
||||
tx.try_send(first).expect("first MRF entry should fill the test channel");
|
||||
|
||||
let admission = queue_mrf_save_admission(&tx, second, "bucket", "second", "test");
|
||||
let admission = queue_mrf_save_entry(&tx, second, "test");
|
||||
tokio::pin!(admission);
|
||||
|
||||
assert!(
|
||||
@@ -2687,6 +2986,30 @@ mod tests {
|
||||
assert!(missing_file.entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_mrf_summary_aggregates_entries_by_bucket_for_obs() {
|
||||
let summary =
|
||||
durable_mrf_backlog_summary_from_sizes([("b1".to_string(), 1024), ("b1".to_string(), 512), ("b2".to_string(), 0)]);
|
||||
|
||||
assert!(summary.available);
|
||||
let buckets = summary
|
||||
.buckets
|
||||
.into_iter()
|
||||
.map(|bucket| (bucket.bucket.clone(), bucket))
|
||||
.collect::<HashMap<_, _>>();
|
||||
assert_eq!(buckets["b1"].count, 2);
|
||||
assert_eq!(buckets["b1"].bytes, 1536);
|
||||
assert_eq!(buckets["b2"].count, 1);
|
||||
assert_eq!(buckets["b2"].bytes, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_mrf_summary_marks_invalid_sizes_unavailable() {
|
||||
let invalid = durable_mrf_backlog_summary_from_sizes([("bucket".to_string(), -1)]);
|
||||
assert!(!invalid.available);
|
||||
assert!(invalid.buckets.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_mrf_snapshot_marks_corrupt_or_invalid_data_unavailable() {
|
||||
let corrupt = durable_mrf_backlog_from_read(Ok(vec![0, 1, 2]));
|
||||
|
||||
@@ -23,8 +23,8 @@ use super::replication_stats_boundary::{
|
||||
};
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicI64, Ordering};
|
||||
use std::sync::{Arc, Mutex as StdMutex};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tokio::time::interval;
|
||||
@@ -143,7 +143,7 @@ pub struct ReplicationStats {
|
||||
// Active worker statistics
|
||||
pub workers: Arc<Mutex<ActiveWorkerStat>>,
|
||||
// Queue statistics cache
|
||||
pub q_cache: Arc<Mutex<QueueCache>>,
|
||||
pub q_cache: Arc<StdMutex<QueueCache>>,
|
||||
// Proxy statistics cache
|
||||
pub p_cache: Arc<Mutex<ProxyStatsCache>>,
|
||||
// MRF backlog statistics (simplified)
|
||||
@@ -158,7 +158,7 @@ impl ReplicationStats {
|
||||
Self {
|
||||
sr_stats: Arc::new(SRStats::new()),
|
||||
workers: Arc::new(Mutex::new(ActiveWorkerStat::new())),
|
||||
q_cache: Arc::new(Mutex::new(QueueCache::new())),
|
||||
q_cache: Arc::new(StdMutex::new(QueueCache::new())),
|
||||
p_cache: Arc::new(Mutex::new(ProxyStatsCache::new())),
|
||||
mrf_stats: HashMap::new(),
|
||||
cache: Arc::new(RwLock::new(HashMap::new())),
|
||||
@@ -198,8 +198,9 @@ impl ReplicationStats {
|
||||
let mut interval = interval(Duration::from_secs(2));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
let mut cache = q_cache_clone.lock().await;
|
||||
cache.update();
|
||||
if let Ok(mut cache) = q_cache_clone.lock() {
|
||||
cache.update();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -391,12 +392,13 @@ impl ReplicationStats {
|
||||
drop(cache);
|
||||
|
||||
{
|
||||
let q_cache = self.q_cache.lock().await;
|
||||
for (bucket, queue_stats) in &q_cache.bucket_stats {
|
||||
let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
|
||||
bucket_stats.q_stat = queue_stats.snapshot();
|
||||
bucket_stats.mark_node_local_provider_available();
|
||||
bucket_stats.queue_scope = ReplicationMetricScope::NodeLocal;
|
||||
if let Ok(q_cache) = self.q_cache.lock() {
|
||||
for (bucket, queue_stats) in &q_cache.bucket_stats {
|
||||
let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
|
||||
bucket_stats.q_stat = queue_stats.snapshot();
|
||||
bucket_stats.mark_node_local_provider_available();
|
||||
bucket_stats.queue_scope = ReplicationMetricScope::NodeLocal;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -429,8 +431,11 @@ impl ReplicationStats {
|
||||
let boot_time = SystemTime::UNIX_EPOCH; // simplified implementation
|
||||
let uptime = SystemTime::now().duration_since(boot_time).unwrap_or_default().as_secs() as i64;
|
||||
|
||||
let q_cache = self.q_cache.lock().await;
|
||||
let queued = q_cache.get_site_stats();
|
||||
let queued = self
|
||||
.q_cache
|
||||
.lock()
|
||||
.map(|q_cache| q_cache.get_site_stats())
|
||||
.unwrap_or_default();
|
||||
|
||||
let p_cache = self.p_cache.lock().await;
|
||||
let proxied = p_cache.get_site_stats();
|
||||
@@ -633,8 +638,9 @@ impl ReplicationStats {
|
||||
drop(cache);
|
||||
|
||||
{
|
||||
let q_cache = self.q_cache.lock().await;
|
||||
if let Some(queue_stats) = q_cache.bucket_stats.get(bucket) {
|
||||
if let Ok(q_cache) = self.q_cache.lock()
|
||||
&& let Some(queue_stats) = q_cache.bucket_stats.get(bucket)
|
||||
{
|
||||
replication_stats.q_stat = queue_stats.snapshot();
|
||||
}
|
||||
}
|
||||
@@ -665,31 +671,17 @@ impl ReplicationStats {
|
||||
}
|
||||
|
||||
/// Increase queue statistics
|
||||
pub async fn inc_q(&self, bucket: &str, size: i64, _is_delete_repl: bool, _op_type: ReplicationType) {
|
||||
let mut q_cache = self.q_cache.lock().await;
|
||||
let stats = q_cache
|
||||
.bucket_stats
|
||||
.entry(bucket.to_string())
|
||||
.or_insert_with(InQueueMetric::default);
|
||||
stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
|
||||
stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
q_cache.sr_queue_stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
|
||||
q_cache.sr_queue_stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
|
||||
pub fn inc_q(&self, bucket: &str, size: i64, _is_delete_repl: bool, _op_type: ReplicationType) {
|
||||
if let Ok(mut q_cache) = self.q_cache.lock() {
|
||||
q_cache.inc(bucket, size);
|
||||
}
|
||||
}
|
||||
|
||||
/// Decrease queue statistics
|
||||
pub async fn dec_q(&self, bucket: &str, size: i64, _is_del_marker: bool, _op_type: ReplicationType) {
|
||||
let mut q_cache = self.q_cache.lock().await;
|
||||
let stats = q_cache
|
||||
.bucket_stats
|
||||
.entry(bucket.to_string())
|
||||
.or_insert_with(InQueueMetric::default);
|
||||
stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
|
||||
stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
|
||||
|
||||
q_cache.sr_queue_stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
|
||||
q_cache.sr_queue_stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
|
||||
pub fn dec_q(&self, bucket: &str, size: i64, _is_del_marker: bool, _op_type: ReplicationType) {
|
||||
if let Ok(mut q_cache) = self.q_cache.lock() {
|
||||
q_cache.dec(bucket, size);
|
||||
}
|
||||
}
|
||||
|
||||
/// Increase proxy metrics
|
||||
@@ -801,14 +793,14 @@ mod tests {
|
||||
async fn latest_stats_include_queue_until_drained() {
|
||||
let stats = ReplicationStats::new();
|
||||
|
||||
stats.inc_q("queued-bucket", 4096, false, ReplicationType::Object).await;
|
||||
stats.inc_q("queued-bucket", 4096, false, ReplicationType::Object);
|
||||
let queued = stats.get_latest_replication_stats("queued-bucket").await;
|
||||
assert!(queued.replication_stats.provider_available);
|
||||
assert_eq!(queued.replication_stats.q_stat.curr.count, 1);
|
||||
assert_eq!(queued.replication_stats.q_stat.curr.bytes, 4096);
|
||||
assert_eq!(queued.replication_stats.queue_scope, ReplicationMetricScope::NodeLocal);
|
||||
|
||||
stats.dec_q("queued-bucket", 4096, false, ReplicationType::Object).await;
|
||||
stats.dec_q("queued-bucket", 4096, false, ReplicationType::Object);
|
||||
let drained = stats.get_latest_replication_stats("queued-bucket").await;
|
||||
assert_eq!(drained.replication_stats.q_stat.curr.count, 0);
|
||||
assert_eq!(drained.replication_stats.q_stat.curr.bytes, 0);
|
||||
@@ -911,7 +903,7 @@ mod tests {
|
||||
for _ in 0..32 {
|
||||
let stats = Arc::clone(&stats);
|
||||
tasks.push(tokio::spawn(async move {
|
||||
stats.inc_q("concurrent-bucket", 7, false, ReplicationType::Object).await;
|
||||
stats.inc_q("concurrent-bucket", 7, false, ReplicationType::Object);
|
||||
}));
|
||||
}
|
||||
for task in tasks {
|
||||
|
||||
@@ -5078,7 +5078,7 @@ impl LocalDisk {
|
||||
Ok((buf, mtime))
|
||||
}
|
||||
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "LocalDisk")]
|
||||
async fn read_metadata_with_dmtime(&self, file_path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
|
||||
check_path_length(file_path.as_ref().to_string_lossy().as_ref())?;
|
||||
|
||||
@@ -5121,7 +5121,7 @@ impl LocalDisk {
|
||||
Ok((data, modtime))
|
||||
}
|
||||
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "LocalDisk")]
|
||||
async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
|
||||
// TODO: timeout support
|
||||
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
|
||||
|
||||
@@ -103,7 +103,7 @@ where
|
||||
/// or `out` is larger than one shard. On error `out`'s contents are
|
||||
/// unspecified but never contain bytes that failed the hash check — the copy
|
||||
/// happens only after verification.
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "BitrotReader")]
|
||||
pub async fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
|
||||
let want = out.len();
|
||||
self.begin_read(want)?;
|
||||
@@ -303,7 +303,7 @@ where
|
||||
|
||||
/// Write a (hash+data) block. Returns the number of data bytes written.
|
||||
/// Returns an error if called after a short write or if data exceeds shard_size.
|
||||
#[hotpath::measure(label = "BitrotWriter::write")]
|
||||
#[hotpath::measure(label = "BitrotWriter::write", impl_type = "BitrotWriter")]
|
||||
pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
if buf.is_empty() {
|
||||
return Ok(0);
|
||||
|
||||
@@ -691,7 +691,7 @@ impl<R> ParallelReader<R>
|
||||
where
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "ParallelReader")]
|
||||
pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
|
||||
// On the reconstruction-verifying GET path, read every live shard reader
|
||||
// in lockstep so all readers advance one block per stripe and stay
|
||||
@@ -1505,7 +1505,7 @@ where
|
||||
}
|
||||
|
||||
impl Erasure {
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub async fn decode<W, R>(
|
||||
&self,
|
||||
writer: &mut W,
|
||||
|
||||
@@ -28,8 +28,11 @@ use std::vec;
|
||||
use tokio::io::AsyncRead;
|
||||
use tokio::runtime::RuntimeFlavor;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::{JoinError, JoinHandle};
|
||||
use tracing::error;
|
||||
|
||||
/// Queue-capacity input for encoded blocks awaiting shard writers; it is not a
|
||||
/// per-PUT or process-RSS memory limit.
|
||||
const ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: &str = "RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES";
|
||||
const ENV_RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS: &str = "RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS";
|
||||
const ENV_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST: &str = "RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST";
|
||||
@@ -87,6 +90,33 @@ fn use_bytesmut_ingest() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST, DEFAULT_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST)
|
||||
})
|
||||
}
|
||||
|
||||
/// Keeps the encoder producer scoped to its parent future. Tokio detaches a
|
||||
/// task when its `JoinHandle` is dropped, so the producer must be aborted when
|
||||
/// an upload is cancelled before the encode pipeline finishes.
|
||||
struct AbortOnDropTask<T>(JoinHandle<T>);
|
||||
|
||||
impl<T> AbortOnDropTask<T> {
|
||||
fn new(task: JoinHandle<T>) -> Self {
|
||||
Self(task)
|
||||
}
|
||||
|
||||
async fn abort_and_wait(&mut self) {
|
||||
self.0.abort();
|
||||
let _ = (&mut self.0).await;
|
||||
}
|
||||
|
||||
async fn join(&mut self) -> Result<T, JoinError> {
|
||||
(&mut self.0).await
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Drop for AbortOnDropTask<T> {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
|
||||
/// Read up to `limit` bytes into `buf`'s uninitialized spare capacity, appending after its
|
||||
/// current length, and distinguish a clean EOF from a short read.
|
||||
///
|
||||
@@ -133,22 +163,65 @@ fn queued_block_bytes(block: &[Bytes]) -> usize {
|
||||
block.iter().map(Bytes::len).sum()
|
||||
}
|
||||
|
||||
async fn drain_queued_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Bytes>>) {
|
||||
while let Some(block) = rx.recv().await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_block_bytes(&block));
|
||||
/// Owns an encoded queue entry's gauge contribution until its consumer takes it.
|
||||
struct QueuedInflightBytes {
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
impl QueuedInflightBytes {
|
||||
fn new(bytes: usize) -> Self {
|
||||
rustfs_io_metrics::add_ec_encode_inflight_bytes(bytes);
|
||||
Self { bytes }
|
||||
}
|
||||
|
||||
fn settle(&mut self) {
|
||||
let bytes = std::mem::take(&mut self.bytes);
|
||||
if bytes != 0 {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for QueuedInflightBytes {
|
||||
fn drop(&mut self) {
|
||||
self.settle();
|
||||
}
|
||||
}
|
||||
|
||||
/// Couples an encoded block with its queue gauge contribution. Keeping the
|
||||
/// guard in the queue entry also covers Tokio sends that complete through a
|
||||
/// permit after the receiver has closed.
|
||||
struct InflightEntry<T> {
|
||||
entry: T,
|
||||
accounting: QueuedInflightBytes,
|
||||
}
|
||||
|
||||
impl<T> InflightEntry<T> {
|
||||
fn new(entry: T, bytes: usize) -> Self {
|
||||
Self {
|
||||
entry,
|
||||
accounting: QueuedInflightBytes::new(bytes),
|
||||
}
|
||||
}
|
||||
|
||||
fn into_inner(mut self) -> T {
|
||||
self.accounting.settle();
|
||||
self.entry
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_queued<T>(
|
||||
sender: &mpsc::Sender<InflightEntry<T>>,
|
||||
entry: T,
|
||||
bytes: usize,
|
||||
) -> Result<(), mpsc::error::SendError<InflightEntry<T>>> {
|
||||
sender.send(InflightEntry::new(entry, bytes)).await
|
||||
}
|
||||
|
||||
fn queued_batch_bytes(batch: &[Vec<Bytes>]) -> usize {
|
||||
batch.iter().map(|block| queued_block_bytes(block)).sum()
|
||||
}
|
||||
|
||||
async fn drain_queued_batched_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Vec<Bytes>>>) {
|
||||
while let Some(batch) = rx.recv().await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
|
||||
}
|
||||
}
|
||||
|
||||
fn dominant_error_summary_label(summary: &WriteQuorumFailureSummary) -> &'static str {
|
||||
summary.dominant_error_label
|
||||
}
|
||||
@@ -504,7 +577,7 @@ impl Erasure {
|
||||
Ok((reader, total))
|
||||
}
|
||||
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub async fn encode<R>(
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
@@ -540,13 +613,14 @@ impl Erasure {
|
||||
));
|
||||
}
|
||||
|
||||
// Bound queued encoded blocks by memory budget to avoid per-request spikes.
|
||||
// Bound queued encoded blocks by a queue budget; this does not bound
|
||||
// reader, encoder, writer, allocator, or process-RSS memory.
|
||||
let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
|
||||
let max_inflight_bytes = erasure_encode_max_inflight_bytes();
|
||||
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
|
||||
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(inflight_blocks);
|
||||
let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Bytes>>>(inflight_blocks);
|
||||
|
||||
let task = tokio::spawn(async move {
|
||||
let mut task = AbortOnDropTask::new(tokio::spawn(async move {
|
||||
let block_size = self.block_size;
|
||||
let mut total = 0;
|
||||
if use_bytesmut_ingest {
|
||||
@@ -567,10 +641,8 @@ impl Erasure {
|
||||
let res = self.clone().encode_block_bytes_mut(encode_buf, n).await?;
|
||||
buf = BytesMut::with_capacity(ingest_capacity);
|
||||
let queued_bytes = queued_block_bytes(&res);
|
||||
rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
|
||||
let send_wait_stage_start = stage_timer_if_enabled();
|
||||
if let Err(err) = tx.send(res).await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
|
||||
if let Err(err) = send_queued(&tx, res, queued_bytes).await {
|
||||
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
|
||||
}
|
||||
record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
|
||||
@@ -598,10 +670,8 @@ impl Erasure {
|
||||
let (res, returned_buf) = self.clone().encode_block(encode_buf, n).await?;
|
||||
buf = returned_buf;
|
||||
let queued_bytes = queued_block_bytes(&res);
|
||||
rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
|
||||
let send_wait_stage_start = stage_timer_if_enabled();
|
||||
if let Err(err) = tx.send(res).await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
|
||||
if let Err(err) = send_queued(&tx, res, queued_bytes).await {
|
||||
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
|
||||
}
|
||||
record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
|
||||
@@ -626,7 +696,7 @@ impl Erasure {
|
||||
}
|
||||
|
||||
Ok((reader, total))
|
||||
});
|
||||
}));
|
||||
|
||||
let mut writers = MultiWriter::new(writers, quorum);
|
||||
|
||||
@@ -638,11 +708,10 @@ impl Erasure {
|
||||
break;
|
||||
};
|
||||
record_internal_stage_if_enabled("erasure_encode_recv_wait", recv_wait_stage_start);
|
||||
let block = block.into_inner();
|
||||
if block.is_empty() {
|
||||
break;
|
||||
}
|
||||
let queued_bytes = queued_block_bytes(&block);
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
|
||||
let write_stage_start = stage_timer_if_enabled();
|
||||
if let Err(err) = writers.write(block).await {
|
||||
write_err = Some(err);
|
||||
@@ -652,9 +721,8 @@ impl Erasure {
|
||||
}
|
||||
|
||||
if let Some(err) = write_err {
|
||||
task.abort();
|
||||
let _ = task.await;
|
||||
drain_queued_inflight_bytes(&mut rx).await;
|
||||
task.abort_and_wait().await;
|
||||
drop(rx);
|
||||
let shutdown_stage_start = stage_timer_if_enabled();
|
||||
if let Err(shutdown_err) = writers.shutdown().await {
|
||||
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
|
||||
@@ -663,14 +731,14 @@ impl Erasure {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let (reader, total) = task.await??;
|
||||
let (reader, total) = task.join().await??;
|
||||
let shutdown_stage_start = stage_timer_if_enabled();
|
||||
writers.shutdown().await?;
|
||||
record_internal_stage_if_enabled("erasure_encode_shutdown", shutdown_stage_start);
|
||||
Ok((reader, total))
|
||||
}
|
||||
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub async fn encode_batched<R>(
|
||||
self: Arc<Self>,
|
||||
mut reader: R,
|
||||
@@ -692,9 +760,9 @@ impl Erasure {
|
||||
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
|
||||
let batch_blocks = encode_batch_block_count().min(inflight_blocks);
|
||||
let channel_capacity = inflight_blocks.div_ceil(batch_blocks).max(1);
|
||||
let (tx, mut rx) = mpsc::channel::<Vec<Vec<Bytes>>>(channel_capacity);
|
||||
let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Vec<Bytes>>>>(channel_capacity);
|
||||
|
||||
let task = tokio::spawn(async move {
|
||||
let mut task = AbortOnDropTask::new(tokio::spawn(async move {
|
||||
let block_size = self.block_size;
|
||||
let mut total = 0;
|
||||
let mut buf = vec![0u8; block_size];
|
||||
@@ -713,10 +781,8 @@ impl Erasure {
|
||||
pending_batch.push(res);
|
||||
|
||||
if pending_batch.len() >= batch_blocks {
|
||||
rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
|
||||
let send_wait_stage_start = stage_timer_if_enabled();
|
||||
if let Err(err) = tx.send(pending_batch).await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
|
||||
if let Err(err) = send_queued(&tx, pending_batch, pending_batch_bytes).await {
|
||||
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
|
||||
}
|
||||
record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
|
||||
@@ -742,17 +808,15 @@ impl Erasure {
|
||||
}
|
||||
|
||||
if !pending_batch.is_empty() {
|
||||
rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
|
||||
let send_wait_stage_start = stage_timer_if_enabled();
|
||||
if let Err(err) = tx.send(pending_batch).await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
|
||||
if let Err(err) = send_queued(&tx, pending_batch, pending_batch_bytes).await {
|
||||
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
|
||||
}
|
||||
record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
|
||||
}
|
||||
|
||||
Ok((reader, total))
|
||||
});
|
||||
}));
|
||||
|
||||
let mut writers = MultiWriter::new(writers, quorum);
|
||||
let mut write_err = None;
|
||||
@@ -763,7 +827,7 @@ impl Erasure {
|
||||
break;
|
||||
};
|
||||
record_internal_stage_if_enabled("erasure_encode_batched_recv_wait", recv_wait_stage_start);
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
|
||||
let batch = batch.into_inner();
|
||||
let write_stage_start = stage_timer_if_enabled();
|
||||
for block in batch {
|
||||
if let Err(err) = writers.write(block).await {
|
||||
@@ -778,9 +842,8 @@ impl Erasure {
|
||||
}
|
||||
|
||||
if let Some(err) = write_err {
|
||||
task.abort();
|
||||
let _ = task.await;
|
||||
drain_queued_batched_inflight_bytes(&mut rx).await;
|
||||
task.abort_and_wait().await;
|
||||
drop(rx);
|
||||
let shutdown_stage_start = stage_timer_if_enabled();
|
||||
if let Err(shutdown_err) = writers.shutdown().await {
|
||||
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
|
||||
@@ -789,7 +852,7 @@ impl Erasure {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let (reader, total) = task.await??;
|
||||
let (reader, total) = task.join().await??;
|
||||
let shutdown_stage_start = stage_timer_if_enabled();
|
||||
writers.shutdown().await?;
|
||||
record_internal_stage_if_enabled("erasure_encode_batched_shutdown", shutdown_stage_start);
|
||||
@@ -798,7 +861,7 @@ impl Erasure {
|
||||
|
||||
/// Fast path for small inline objects: skip tokio::spawn + mpsc channel.
|
||||
/// Reads all data, encodes directly, writes shards sequentially.
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub async fn encode_inline_small<R>(
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
@@ -813,7 +876,7 @@ impl Erasure {
|
||||
|
||||
/// Fast path for single-block non-inline objects: avoids the producer/consumer
|
||||
/// pipeline in `encode()` while keeping the same writer/quorum/shutdown semantics.
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub async fn encode_single_block_non_inline<R>(
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
@@ -839,7 +902,104 @@ mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncWrite, AsyncWriteExt};
|
||||
use tokio::io::{AsyncWrite, AsyncWriteExt, ReadBuf};
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
struct PendingReader {
|
||||
entered: Option<oneshot::Sender<()>>,
|
||||
dropped: Option<oneshot::Sender<()>>,
|
||||
}
|
||||
|
||||
impl PendingReader {
|
||||
fn new() -> (Self, oneshot::Receiver<()>, oneshot::Receiver<()>) {
|
||||
let (entered_tx, entered_rx) = oneshot::channel();
|
||||
let (dropped_tx, dropped_rx) = oneshot::channel();
|
||||
(
|
||||
Self {
|
||||
entered: Some(entered_tx),
|
||||
dropped: Some(dropped_tx),
|
||||
},
|
||||
entered_rx,
|
||||
dropped_rx,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for PendingReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
if let Some(entered) = self.entered.take() {
|
||||
let _ = entered.send(());
|
||||
}
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PendingReader {
|
||||
fn drop(&mut self) {
|
||||
if let Some(dropped) = self.dropped.take() {
|
||||
let _ = dropped.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct BlocksThenPendingReader {
|
||||
blocks_remaining: usize,
|
||||
block: Vec<u8>,
|
||||
blocked: Option<oneshot::Sender<()>>,
|
||||
dropped: Option<oneshot::Sender<()>>,
|
||||
final_block: Option<oneshot::Sender<()>>,
|
||||
}
|
||||
|
||||
impl BlocksThenPendingReader {
|
||||
fn new(
|
||||
blocks_remaining: usize,
|
||||
block_size: usize,
|
||||
) -> (Self, oneshot::Receiver<()>, oneshot::Receiver<()>, oneshot::Receiver<()>) {
|
||||
let (blocked_tx, blocked_rx) = oneshot::channel();
|
||||
let (dropped_tx, dropped_rx) = oneshot::channel();
|
||||
let (final_block_tx, final_block_rx) = oneshot::channel();
|
||||
(
|
||||
Self {
|
||||
blocks_remaining,
|
||||
block: vec![0x5a; block_size],
|
||||
blocked: Some(blocked_tx),
|
||||
dropped: Some(dropped_tx),
|
||||
final_block: Some(final_block_tx),
|
||||
},
|
||||
blocked_rx,
|
||||
dropped_rx,
|
||||
final_block_rx,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for BlocksThenPendingReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
if self.blocks_remaining == 0 {
|
||||
if let Some(blocked) = self.blocked.take() {
|
||||
let _ = blocked.send(());
|
||||
}
|
||||
return Poll::Pending;
|
||||
}
|
||||
|
||||
if self.blocks_remaining == 1
|
||||
&& let Some(final_block) = self.final_block.take()
|
||||
{
|
||||
let _ = final_block.send(());
|
||||
}
|
||||
self.blocks_remaining -= 1;
|
||||
buf.put_slice(&self.block);
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BlocksThenPendingReader {
|
||||
fn drop(&mut self) {
|
||||
if let Some(dropped) = self.dropped.take() {
|
||||
let _ = dropped.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn erasure_with_zero_block_size() -> Erasure {
|
||||
let mut erasure = Erasure::default();
|
||||
@@ -897,6 +1057,54 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct FailAfterReaderBlocksWriter {
|
||||
reader_blocked: oneshot::Receiver<()>,
|
||||
writes: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
impl AsyncWrite for FailAfterReaderBlocksWriter {
|
||||
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
match Pin::new(&mut self.reader_blocked).poll(cx) {
|
||||
Poll::Pending => Poll::Pending,
|
||||
Poll::Ready(_) => {
|
||||
self.writes.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Poll::Ready(Err(std::io::Error::other("injected write failure after producer blocks")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
struct StallOnWriteWithSignal {
|
||||
entered: Option<oneshot::Sender<()>>,
|
||||
writes: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
impl AsyncWrite for StallOnWriteWithSignal {
|
||||
fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
self.writes.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
if let Some(entered) = self.entered.take() {
|
||||
let _ = entered.send(());
|
||||
}
|
||||
Poll::Pending
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ShortWriteWriter;
|
||||
|
||||
@@ -1046,6 +1254,202 @@ mod tests {
|
||||
BitrotWriterWrapper::new(CustomWriter::new_tokio_writer(writer), shard_size, HashAlgorithm::None)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum EncodePipeline {
|
||||
Vec,
|
||||
BytesMut,
|
||||
Batched,
|
||||
}
|
||||
|
||||
async fn aborting_encode_drops_blocked_producer(pipeline: EncodePipeline) {
|
||||
const BLOCK_SIZE: usize = 16;
|
||||
|
||||
let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
|
||||
let committed = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut writers = vec![Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE))];
|
||||
let (reader, entered, dropped) = PendingReader::new();
|
||||
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
|
||||
|
||||
let encode = match pipeline {
|
||||
EncodePipeline::Vec => {
|
||||
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await })
|
||||
}
|
||||
EncodePipeline::BytesMut => {
|
||||
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await })
|
||||
}
|
||||
EncodePipeline::Batched => tokio::spawn(async move { erasure.encode_batched(reader, &mut writers, 1).await }),
|
||||
};
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), entered)
|
||||
.await
|
||||
.expect("producer should enter the blocked reader before cancellation")
|
||||
.expect("blocked reader should signal entry");
|
||||
encode.abort();
|
||||
assert!(matches!(encode.await, Err(err) if err.is_cancelled()), "encode task should be cancelled");
|
||||
tokio::time::timeout(Duration::from_secs(1), dropped)
|
||||
.await
|
||||
.expect("cancelling encode should drop the producer reader")
|
||||
.expect("blocked reader should signal producer drop");
|
||||
assert!(
|
||||
committed.lock().expect("committed buffer should be lockable").is_empty(),
|
||||
"cancelling before the first encoded block must not make data visible"
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
gauge_baseline,
|
||||
"cancelling the encode pipeline must preserve the inflight queue gauge"
|
||||
);
|
||||
}
|
||||
|
||||
async fn writer_error_aborts_blocked_producer(pipeline: EncodePipeline) {
|
||||
const BLOCK_SIZE: usize = 16;
|
||||
|
||||
let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
|
||||
let blocks_before_pending = match pipeline {
|
||||
EncodePipeline::Batched => encode_batch_block_count(),
|
||||
EncodePipeline::Vec | EncodePipeline::BytesMut => 1,
|
||||
};
|
||||
let (reader, reader_blocked, reader_dropped, _final_block) =
|
||||
BlocksThenPendingReader::new(blocks_before_pending, BLOCK_SIZE);
|
||||
let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let mut writers = vec![Some(bitrot_writer(
|
||||
FailAfterReaderBlocksWriter {
|
||||
reader_blocked,
|
||||
writes: writes.clone(),
|
||||
},
|
||||
BLOCK_SIZE,
|
||||
))];
|
||||
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
|
||||
|
||||
let result = match pipeline {
|
||||
EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await,
|
||||
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await,
|
||||
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
|
||||
};
|
||||
|
||||
let err = match result {
|
||||
Ok(_) => panic!("writer quorum failure should fail the encode pipeline"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(err.to_string().contains("Failed to write data"));
|
||||
tokio::time::timeout(Duration::from_secs(1), reader_dropped)
|
||||
.await
|
||||
.expect("writer failure should abort the blocked producer")
|
||||
.expect("blocked producer should signal reader drop");
|
||||
assert_eq!(
|
||||
writes.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"writer failure must stop the pipeline before any additional shard write"
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
gauge_baseline,
|
||||
"writer failure must settle all queued and pending encoded bytes"
|
||||
);
|
||||
}
|
||||
|
||||
async fn aborting_full_queue_settles_pending_send() {
|
||||
const BLOCK_SIZE: usize = 16;
|
||||
|
||||
let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
|
||||
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
|
||||
let inflight_blocks = encode_channel_capacity(
|
||||
erasure.shard_size().saturating_mul(erasure.total_shard_count()),
|
||||
erasure_encode_max_inflight_bytes(),
|
||||
);
|
||||
let (reader, _reader_blocked, reader_dropped, final_block) =
|
||||
BlocksThenPendingReader::new(inflight_blocks + 2, BLOCK_SIZE);
|
||||
let (writer_entered_tx, writer_entered) = oneshot::channel();
|
||||
let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let mut writers = vec![Some(bitrot_writer(
|
||||
StallOnWriteWithSignal {
|
||||
entered: Some(writer_entered_tx),
|
||||
writes: writes.clone(),
|
||||
},
|
||||
BLOCK_SIZE,
|
||||
))];
|
||||
let erasure_for_task = erasure.clone();
|
||||
let encode = tokio::spawn(async move { erasure_for_task.encode_with_ingest_mode(reader, &mut writers, 1, false).await });
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), writer_entered)
|
||||
.await
|
||||
.expect("consumer should start the first writer call")
|
||||
.expect("stalling writer should signal entry");
|
||||
tokio::time::timeout(Duration::from_secs(1), final_block)
|
||||
.await
|
||||
.expect("producer should supply the block whose send fills the queue")
|
||||
.expect("reader should signal final block");
|
||||
|
||||
let expected_queued_bytes =
|
||||
u64::try_from((inflight_blocks + 1) * BLOCK_SIZE).expect("queued byte count should fit the gauge");
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while rustfs_io_metrics::current_ec_encode_inflight_bytes() < gauge_baseline + expected_queued_bytes {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("producer should account for the pending send after the queue fills");
|
||||
|
||||
encode.abort();
|
||||
assert!(matches!(encode.await, Err(err) if err.is_cancelled()), "encode task should be cancelled");
|
||||
tokio::time::timeout(Duration::from_secs(1), reader_dropped)
|
||||
.await
|
||||
.expect("cancelling a full queue should abort its producer")
|
||||
.expect("full-queue producer should signal reader drop");
|
||||
assert_eq!(
|
||||
writes.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"cancellation must not resume the stalled writer"
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
gauge_baseline,
|
||||
"cancelling a full queue must settle queued and pending bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cancelling_vec_encode_drops_blocked_producer() {
|
||||
aborting_encode_drops_blocked_producer(EncodePipeline::Vec).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cancelling_bytesmut_encode_drops_blocked_producer() {
|
||||
aborting_encode_drops_blocked_producer(EncodePipeline::BytesMut).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cancelling_batched_encode_drops_blocked_producer() {
|
||||
aborting_encode_drops_blocked_producer(EncodePipeline::Batched).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn vec_writer_error_aborts_blocked_producer() {
|
||||
writer_error_aborts_blocked_producer(EncodePipeline::Vec).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn bytesmut_writer_error_aborts_blocked_producer() {
|
||||
writer_error_aborts_blocked_producer(EncodePipeline::BytesMut).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn batched_writer_error_aborts_blocked_producer() {
|
||||
writer_error_aborts_blocked_producer(EncodePipeline::Batched).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cancelling_full_queue_settles_pending_send() {
|
||||
aborting_full_queue_settles_pending_send().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn helper_writers_cover_flush_and_shutdown_paths() {
|
||||
let mut failing_write = FailingWriteWriter;
|
||||
@@ -1329,25 +1733,99 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drain_queued_inflight_bytes_consumes_pending_blocks() {
|
||||
let (tx, mut rx) = mpsc::channel(2);
|
||||
tx.send(vec![Bytes::from_static(b"queued")]).await.unwrap();
|
||||
drop(tx);
|
||||
#[serial_test::serial]
|
||||
async fn queued_inflight_bytes_are_settled_on_all_queue_exit_paths() {
|
||||
let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
|
||||
let (tx, rx) = mpsc::channel(1);
|
||||
let queued = vec![Bytes::from_static(b"queued")];
|
||||
let queued_bytes = queued_block_bytes(&queued);
|
||||
|
||||
drain_queued_inflight_bytes(&mut rx).await;
|
||||
send_queued(&tx, queued, queued_bytes)
|
||||
.await
|
||||
.expect("first queue entry should fit");
|
||||
|
||||
assert!(rx.recv().await.is_none());
|
||||
let blocked = vec![Bytes::from_static(b"blocked")];
|
||||
let blocked_bytes = queued_block_bytes(&blocked);
|
||||
{
|
||||
let pending_send = send_queued(&tx, blocked, blocked_bytes);
|
||||
tokio::pin!(pending_send);
|
||||
assert!(
|
||||
futures::poll!(pending_send.as_mut()).is_pending(),
|
||||
"full queue must suspend producer send"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
baseline + u64::try_from(queued_bytes).expect("queue bytes fit the gauge"),
|
||||
"dropping a pending producer send must compensate its bytes"
|
||||
);
|
||||
|
||||
drop(rx);
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
baseline,
|
||||
"dropping the receiver must settle every buffered entry"
|
||||
);
|
||||
|
||||
let rejected = vec![Bytes::from_static(b"rejected")];
|
||||
let rejected_bytes = queued_block_bytes(&rejected);
|
||||
assert!(
|
||||
send_queued(&tx, rejected, rejected_bytes).await.is_err(),
|
||||
"closed receiver must reject a new send"
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
baseline,
|
||||
"failed sends must compensate their bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drain_queued_batched_inflight_bytes_consumes_pending_batches() {
|
||||
let (tx, mut rx) = mpsc::channel(2);
|
||||
tx.send(vec![vec![Bytes::from_static(b"queued")]]).await.unwrap();
|
||||
drop(tx);
|
||||
#[serial_test::serial]
|
||||
async fn queued_batch_entry_settles_bytes_before_handoff() {
|
||||
let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
|
||||
let (tx, rx) = mpsc::channel(2);
|
||||
let mut rx = rx;
|
||||
let batch = vec![vec![Bytes::from_static(b"queued")], vec![Bytes::from_static(b"batch")]];
|
||||
let batch_bytes = queued_batch_bytes(&batch);
|
||||
|
||||
drain_queued_batched_inflight_bytes(&mut rx).await;
|
||||
send_queued(&tx, batch, batch_bytes).await.expect("batch should be queued");
|
||||
let batch = rx.recv().await.expect("queued batch should be received").into_inner();
|
||||
assert_eq!(batch_bytes, queued_batch_bytes(&batch));
|
||||
|
||||
assert!(rx.recv().await.is_none());
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
baseline,
|
||||
"receiving a batch must settle all contained block bytes before shard writes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn queued_entry_settles_when_a_closed_receiver_accepts_an_outstanding_permit() {
|
||||
let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
|
||||
let (tx, mut rx) = mpsc::channel(1);
|
||||
let permit = tx
|
||||
.clone()
|
||||
.reserve_owned()
|
||||
.await
|
||||
.expect("open receiver should reserve queue capacity");
|
||||
rx.close();
|
||||
assert!(
|
||||
matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
|
||||
"an outstanding permit must leave the closed queue observably empty"
|
||||
);
|
||||
|
||||
let block = vec![Bytes::from_static(b"late-permit")];
|
||||
let block_bytes = queued_block_bytes(&block);
|
||||
permit.send(InflightEntry::new(block, block_bytes));
|
||||
drop(rx);
|
||||
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
baseline,
|
||||
"a queue entry sent through an outstanding permit must settle when Tokio drops it"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -640,7 +640,7 @@ impl Erasure {
|
||||
/// # Returns
|
||||
/// A vector of encoded shards as `Bytes`.
|
||||
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> {
|
||||
let shard_size_fn = if self.uses_legacy {
|
||||
calc_shard_size_legacy
|
||||
@@ -688,7 +688,7 @@ impl Erasure {
|
||||
|
||||
/// Encode owned data, avoiding a copy when the caller already has a heap buffer.
|
||||
/// Falls back to copying into a new buffer if zero-copy conversion fails.
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub fn encode_data_owned(&self, data: Vec<u8>) -> io::Result<Vec<Bytes>> {
|
||||
let shard_size_fn = if self.uses_legacy {
|
||||
calc_shard_size_legacy
|
||||
@@ -752,7 +752,7 @@ impl Erasure {
|
||||
/// block), the `resize(need_total_size)` below stays within capacity for every
|
||||
/// `data_len <= block_size` — both shard-size formulas are monotone in
|
||||
/// `data_len` — so this function never reallocates the buffer.
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub fn encode_data_bytes_mut(&self, mut data_buffer: BytesMut, data_len: usize) -> io::Result<Vec<Bytes>> {
|
||||
let shard_size_fn = if self.uses_legacy {
|
||||
calc_shard_size_legacy
|
||||
@@ -805,7 +805,7 @@ impl Erasure {
|
||||
///
|
||||
/// # Returns
|
||||
/// Ok if reconstruction succeeds, error otherwise.
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
|
||||
if self.parity_shards > 0 {
|
||||
if self.uses_legacy {
|
||||
@@ -825,7 +825,7 @@ impl Erasure {
|
||||
}
|
||||
|
||||
/// Decode and reconstruct missing data shards, then regenerate parity shards.
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
pub fn decode_data_and_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
|
||||
if self.parity_shards > 0 {
|
||||
if self.uses_legacy {
|
||||
|
||||
@@ -19,6 +19,8 @@ use crate::diagnostics::get::{
|
||||
GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
|
||||
GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
#[cfg(feature = "hotpath")]
|
||||
use crate::disk::FileWriter;
|
||||
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
|
||||
use crate::erasure::coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
|
||||
use bytes::Bytes;
|
||||
@@ -36,6 +38,11 @@ use std::time::Instant;
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tracing::debug;
|
||||
|
||||
#[cfg(all(test, feature = "hotpath"))]
|
||||
tokio::task_local! {
|
||||
static FORCE_MMAP_COPY_FAILURE_FOR_TEST: ();
|
||||
}
|
||||
|
||||
/// A shard source for the bitrot reader.
|
||||
///
|
||||
/// `InMemory` keeps the `Bytes` concrete instead of erasing it behind
|
||||
@@ -360,10 +367,21 @@ async fn open_disk_reader(
|
||||
mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
|
||||
direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
|
||||
});
|
||||
match disk
|
||||
.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
|
||||
.await
|
||||
{
|
||||
let mmap_result = {
|
||||
#[cfg(all(test, feature = "hotpath"))]
|
||||
if FORCE_MMAP_COPY_FAILURE_FOR_TEST.try_with(|_| ()).is_ok() {
|
||||
Err(DiskError::other("forced mmap-copy failure for test"))
|
||||
} else {
|
||||
disk.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
|
||||
.await
|
||||
}
|
||||
#[cfg(not(all(test, feature = "hotpath")))]
|
||||
{
|
||||
disk.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
|
||||
.await
|
||||
}
|
||||
};
|
||||
match mmap_result {
|
||||
Ok(bytes) => {
|
||||
let duration_ms = zero_copy_start.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
@@ -398,7 +416,11 @@ async fn open_disk_reader(
|
||||
}
|
||||
|
||||
return match stream_result {
|
||||
Ok(reader) => Ok(wrap_first_read_metrics(reader, metrics_path)),
|
||||
Ok(reader) => {
|
||||
#[cfg(feature = "hotpath")]
|
||||
let reader = instrument_raw_shard_reader(reader, disk.is_local());
|
||||
Ok(wrap_first_read_metrics(reader, metrics_path))
|
||||
}
|
||||
Err(_) => Err(err),
|
||||
};
|
||||
}
|
||||
@@ -407,6 +429,8 @@ async fn open_disk_reader(
|
||||
|
||||
let stream_start = stage_metrics_enabled.then(Instant::now);
|
||||
let reader = disk.read_file_stream(bucket, path, offset, length).await?;
|
||||
#[cfg(feature = "hotpath")]
|
||||
let reader = instrument_raw_shard_reader(reader, disk.is_local());
|
||||
if let Some(metrics_path) = metrics_path {
|
||||
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_STREAM, stream_start);
|
||||
}
|
||||
@@ -427,6 +451,37 @@ fn wrap_first_read_metrics(reader: FileReader, metrics_path: Option<&'static str
|
||||
ShardReader::Stream(reader)
|
||||
}
|
||||
|
||||
// The labels are deliberately fixed: object keys, disk paths, and remote hosts
|
||||
// are all high-cardinality or sensitive and belong nowhere in a profiling report.
|
||||
#[cfg(feature = "hotpath")]
|
||||
const RAW_SHARD_READ_LOCAL_LABEL: &str = "EC raw shard read local";
|
||||
#[cfg(feature = "hotpath")]
|
||||
const RAW_SHARD_READ_REMOTE_LABEL: &str = "EC raw shard read remote";
|
||||
#[cfg(feature = "hotpath")]
|
||||
const RAW_SHARD_WRITE_LOCAL_LABEL: &str = "EC raw shard write local";
|
||||
#[cfg(feature = "hotpath")]
|
||||
const RAW_SHARD_WRITE_REMOTE_LABEL: &str = "EC raw shard write remote";
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
fn instrument_raw_shard_reader(reader: FileReader, is_local: bool) -> FileReader {
|
||||
// `io!` aggregates by call site, so the local and remote branches must remain distinct.
|
||||
if is_local {
|
||||
Box::new(hotpath::io!(reader, label = RAW_SHARD_READ_LOCAL_LABEL))
|
||||
} else {
|
||||
Box::new(hotpath::io!(reader, label = RAW_SHARD_READ_REMOTE_LABEL))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
fn instrument_raw_shard_writer(writer: FileWriter, is_local: bool) -> FileWriter {
|
||||
// `io!` aggregates by call site, so the local and remote branches must remain distinct.
|
||||
if is_local {
|
||||
Box::new(hotpath::io!(writer, label = RAW_SHARD_WRITE_LOCAL_LABEL))
|
||||
} else {
|
||||
Box::new(hotpath::io!(writer, label = RAW_SHARD_WRITE_REMOTE_LABEL))
|
||||
}
|
||||
}
|
||||
|
||||
fn bitrot_encoded_range(offset: usize, length: usize, shard_size: usize, checksum_algo: HashAlgorithm) -> (usize, usize) {
|
||||
(
|
||||
offset.div_ceil(shard_size) * checksum_algo.size() + offset,
|
||||
@@ -686,6 +741,8 @@ pub async fn create_bitrot_writer(
|
||||
};
|
||||
|
||||
let file = disk.create_file("", volume, path, length).await?;
|
||||
#[cfg(feature = "hotpath")]
|
||||
let file = instrument_raw_shard_writer(file, disk.is_local());
|
||||
CustomWriter::new_tokio_writer(file)
|
||||
} else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
@@ -698,6 +755,182 @@ pub async fn create_bitrot_writer(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
use crate::cluster::rpc::RemoteDisk;
|
||||
#[cfg(feature = "hotpath")]
|
||||
use crate::cluster::rpc::internode_data_transport::{
|
||||
InternodeDataTransport, InternodeDataTransportCapabilities, ReadStreamRequest, WalkDirStreamRequest, WriteStreamRequest,
|
||||
};
|
||||
#[cfg(feature = "hotpath")]
|
||||
use crate::disk::{Disk, DiskOption, error::Result};
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
#[derive(Debug, Clone, Default)]
|
||||
struct TestRemoteDataTransport {
|
||||
bytes: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
impl TestRemoteDataTransport {
|
||||
fn bytes(&self) -> Vec<u8> {
|
||||
self.bytes
|
||||
.lock()
|
||||
.expect("test remote transport bytes lock should not be poisoned")
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
#[derive(Debug)]
|
||||
struct TestRemoteWriter {
|
||||
bytes: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
impl tokio::io::AsyncWrite for TestRemoteWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
|
||||
self.bytes
|
||||
.lock()
|
||||
.expect("test remote transport bytes lock should not be poisoned")
|
||||
.extend_from_slice(buf);
|
||||
Poll::Ready(Ok(buf.len()))
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
#[async_trait::async_trait]
|
||||
impl InternodeDataTransport for TestRemoteDataTransport {
|
||||
async fn open_read(&self, _request: ReadStreamRequest) -> Result<FileReader> {
|
||||
Ok(Box::new(Cursor::new(self.bytes())))
|
||||
}
|
||||
|
||||
async fn open_write(&self, _request: WriteStreamRequest) -> Result<FileWriter> {
|
||||
Ok(Box::new(TestRemoteWriter {
|
||||
bytes: Arc::clone(&self.bytes),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result<FileReader> {
|
||||
panic!("open_walk_dir must not be used by the raw shard I/O test")
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
"bitrot-test-remote"
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> InternodeDataTransportCapabilities {
|
||||
InternodeDataTransportCapabilities::tcp_http()
|
||||
}
|
||||
}
|
||||
|
||||
async fn local_test_disk() -> (DiskStore, tempfile::TempDir) {
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::{DiskOption, new_disk};
|
||||
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("local disk should be created");
|
||||
|
||||
(disk, dir)
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
async fn remote_test_disk() -> (DiskStore, TestRemoteDataTransport) {
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
|
||||
let endpoint = Endpoint {
|
||||
url: url::Url::parse("http://remote-node:9000/data/rustfs0").expect("test remote endpoint should parse"),
|
||||
is_local: false,
|
||||
pool_idx: 0,
|
||||
set_idx: 0,
|
||||
disk_idx: 0,
|
||||
};
|
||||
let transport = TestRemoteDataTransport::default();
|
||||
let remote = RemoteDisk::new(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
Arc::new(transport.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("test remote disk should be created");
|
||||
|
||||
(Arc::new(Disk::Remote(Box::new(remote))), transport)
|
||||
}
|
||||
|
||||
async fn round_trip_disk_bitrot(disk: &DiskStore, bucket: &str, path: &str, payload: &[u8], shard_size: usize) -> Vec<u8> {
|
||||
disk.make_volume(bucket).await.expect("volume should be created");
|
||||
let mut writer = create_bitrot_writer(
|
||||
false,
|
||||
Some(disk),
|
||||
bucket,
|
||||
path,
|
||||
i64::try_from(payload.len()).expect("test payload length should fit i64"),
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
)
|
||||
.await
|
||||
.expect("disk bitrot writer should open the raw shard file");
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
writer
|
||||
.write(chunk)
|
||||
.await
|
||||
.expect("disk bitrot writer should preserve each shard block");
|
||||
}
|
||||
writer.shutdown().await.expect("disk bitrot writer should close cleanly");
|
||||
|
||||
let mut reader = create_bitrot_reader(
|
||||
None,
|
||||
Some(disk),
|
||||
bucket,
|
||||
path,
|
||||
0,
|
||||
payload.len(),
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("disk bitrot reader should open the raw shard file")
|
||||
.expect("disk bitrot reader should exist");
|
||||
let mut actual = Vec::with_capacity(payload.len());
|
||||
while actual.len() < payload.len() {
|
||||
let remaining = payload.len() - actual.len();
|
||||
let mut chunk = vec![0; remaining.min(shard_size)];
|
||||
let read = reader
|
||||
.read(&mut chunk)
|
||||
.await
|
||||
.expect("disk bitrot reader should return the complete shard body");
|
||||
assert!(read > 0, "disk bitrot reader must not end before the expected shard body is complete");
|
||||
actual.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
|
||||
actual
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_mmap_read_enabled_accepts_legacy_zero_copy_alias() {
|
||||
temp_env::with_vars(
|
||||
@@ -724,6 +957,165 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
#[test]
|
||||
fn raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes() {
|
||||
const CHILD_ENV: &str = "RUSTFS_HOTPATH_RAW_SHARD_IO_TEST_CHILD";
|
||||
if std::env::var_os(CHILD_ENV).is_none() {
|
||||
let status = std::process::Command::new(std::env::current_exe().expect("test executable path should be available"))
|
||||
.arg("--exact")
|
||||
.arg("io_support::bitrot::tests::raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes")
|
||||
.arg("--nocapture")
|
||||
.env(CHILD_ENV, "1")
|
||||
.status()
|
||||
.expect("isolated HotPath I/O test process should start");
|
||||
assert!(status.success(), "isolated HotPath I/O test process should pass");
|
||||
return;
|
||||
}
|
||||
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should be created")
|
||||
.block_on(raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes_in_isolated_process());
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
async fn raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes_in_isolated_process() {
|
||||
use hotpath::{Format, HotpathGuardBuilder, Section};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let report_dir = tempfile::tempdir().expect("report tempdir should be created");
|
||||
let report_path = report_dir.path().join("hotpath-io.json");
|
||||
let guard = HotpathGuardBuilder::new("raw_shard_io_test")
|
||||
.format(Format::Json)
|
||||
.output_path(&report_path)
|
||||
.sections(vec![Section::Io])
|
||||
.build();
|
||||
|
||||
let (disk, _dir) = local_test_disk().await;
|
||||
let bucket = "test-bucket";
|
||||
let path = "obj/hotpath-part.1";
|
||||
let payload = b"local shard bytes";
|
||||
let shard_size = 4;
|
||||
let local_read = round_trip_disk_bitrot(&disk, bucket, path, payload, shard_size).await;
|
||||
assert_eq!(local_read, payload, "local raw shard I/O instrumentation must not alter stored bytes");
|
||||
|
||||
let fallback_path = "obj/hotpath-mmap-fallback-part.1";
|
||||
let fallback_payload = b"mmap fallback shard bytes";
|
||||
disk.write_all("test-bucket", fallback_path, Bytes::from_static(fallback_payload))
|
||||
.await
|
||||
.expect("fallback shard file should be written");
|
||||
let mut fallback_reader = FORCE_MMAP_COPY_FAILURE_FOR_TEST
|
||||
.scope((), open_disk_reader(&disk, bucket, fallback_path, 0, fallback_payload.len(), true, None))
|
||||
.await
|
||||
.expect("mmap-copy failure should fall back to a raw shard stream");
|
||||
assert!(
|
||||
matches!(fallback_reader, ShardReader::Stream(_)),
|
||||
"forced mmap-copy failure must use the streaming fallback"
|
||||
);
|
||||
let mut fallback_read = Vec::new();
|
||||
fallback_reader
|
||||
.read_to_end(&mut fallback_read)
|
||||
.await
|
||||
.expect("mmap-copy fallback stream should preserve bytes");
|
||||
assert_eq!(fallback_read, fallback_payload);
|
||||
|
||||
let (remote_disk, remote_transport) = remote_test_disk().await;
|
||||
let remote_payload = b"remote shard bytes";
|
||||
let mut remote_writer = create_bitrot_writer(
|
||||
false,
|
||||
Some(&remote_disk),
|
||||
bucket,
|
||||
"obj/hotpath-remote-part.1",
|
||||
i64::try_from(remote_payload.len()).expect("remote payload length should fit i64"),
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
)
|
||||
.await
|
||||
.expect("remote bitrot writer should use the production raw writer path");
|
||||
for chunk in remote_payload.chunks(shard_size) {
|
||||
remote_writer
|
||||
.write(chunk)
|
||||
.await
|
||||
.expect("remote bitrot writer should preserve bytes");
|
||||
}
|
||||
remote_writer
|
||||
.shutdown()
|
||||
.await
|
||||
.expect("remote bitrot writer should close cleanly");
|
||||
assert_eq!(remote_transport.bytes(), remote_payload);
|
||||
|
||||
let mut reader = create_bitrot_reader(
|
||||
None,
|
||||
Some(&remote_disk),
|
||||
bucket,
|
||||
"obj/hotpath-remote-part.1",
|
||||
0,
|
||||
remote_payload.len(),
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("remote bitrot reader should use the production raw reader path")
|
||||
.expect("remote bitrot reader should exist");
|
||||
let mut remote_read = Vec::with_capacity(remote_payload.len());
|
||||
while remote_read.len() < remote_payload.len() {
|
||||
let remaining = remote_payload.len() - remote_read.len();
|
||||
let mut chunk = vec![0; remaining.min(shard_size)];
|
||||
let read = reader
|
||||
.read(&mut chunk)
|
||||
.await
|
||||
.expect("remote bitrot reader should preserve bytes");
|
||||
assert!(read > 0, "remote bitrot reader must not end before the expected shard body is complete");
|
||||
remote_read.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
assert_eq!(remote_read, remote_payload);
|
||||
|
||||
drop(guard);
|
||||
let report = std::fs::read_to_string(&report_path).expect("HotPath I/O report should be written");
|
||||
let report: serde_json::Value = serde_json::from_str(&report).expect("HotPath I/O report should be valid JSON");
|
||||
let entries = report["io"]["data"]
|
||||
.as_array()
|
||||
.expect("HotPath I/O report should include data rows");
|
||||
let io_bytes = |label: &str, direction: &str| {
|
||||
let byte_count: u64 = entries
|
||||
.iter()
|
||||
.filter(|entry| entry["label"].as_str().is_some_and(|entry_label| entry_label == label))
|
||||
.filter_map(|entry| entry[direction]["bytes"].as_u64())
|
||||
.sum();
|
||||
assert!(byte_count > 0, "report must include fixed label {label}");
|
||||
byte_count
|
||||
};
|
||||
let payload_len = u64::try_from(payload.len()).expect("test payload length should fit u64");
|
||||
assert_eq!(
|
||||
io_bytes(RAW_SHARD_READ_LOCAL_LABEL, "read"),
|
||||
payload_len + u64::try_from(fallback_payload.len()).expect("fallback payload length should fit u64")
|
||||
);
|
||||
assert_eq!(io_bytes(RAW_SHARD_WRITE_LOCAL_LABEL, "write"), payload_len);
|
||||
assert_eq!(
|
||||
io_bytes(RAW_SHARD_READ_REMOTE_LABEL, "read"),
|
||||
u64::try_from(remote_payload.len()).expect("remote payload length should fit u64")
|
||||
);
|
||||
assert_eq!(
|
||||
io_bytes(RAW_SHARD_WRITE_REMOTE_LABEL, "write"),
|
||||
u64::try_from(remote_payload.len()).expect("remote payload length should fit u64")
|
||||
);
|
||||
for label in [
|
||||
RAW_SHARD_READ_LOCAL_LABEL,
|
||||
RAW_SHARD_READ_REMOTE_LABEL,
|
||||
RAW_SHARD_WRITE_LOCAL_LABEL,
|
||||
RAW_SHARD_WRITE_REMOTE_LABEL,
|
||||
] {
|
||||
assert!(
|
||||
!label.contains(['/', ':', '?', '@']),
|
||||
"raw shard I/O labels must not carry a path, host, query, or credential delimiter: {label}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_mmap_read_max_length_defaults_and_env_override() {
|
||||
temp_env::with_var(ENV_OBJECT_MMAP_READ_MAX_LENGTH, None::<&str>, || {
|
||||
@@ -741,25 +1133,9 @@ mod tests {
|
||||
// be materialized in memory by the mmap-copy path; over-cap reads stream.
|
||||
#[tokio::test]
|
||||
async fn open_disk_reader_streams_when_length_exceeds_mmap_cap() {
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::{DiskOption, new_disk};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("local disk should be created");
|
||||
let (disk, _dir) = local_test_disk().await;
|
||||
|
||||
let payload = vec![7u8; 4096];
|
||||
disk.make_volume("test-bucket").await.expect("volume should be created");
|
||||
@@ -814,6 +1190,17 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_bitrot_reader_and_writer_preserve_full_shard_body() {
|
||||
let (disk, _dir) = local_test_disk().await;
|
||||
let bucket = "test-bucket";
|
||||
let path = "obj/wrapped-part.1";
|
||||
let payload = b"wrapped shard body";
|
||||
let actual = round_trip_disk_bitrot(&disk, bucket, path, payload, 4).await;
|
||||
|
||||
assert_eq!(actual, payload, "raw shard I/O instrumentation must not alter stored bytes");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_reader_with_inline_data() {
|
||||
let test_data = b"hello world test data";
|
||||
|
||||
@@ -199,7 +199,7 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
pub async fn read_version_optimized(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -238,7 +238,7 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
pub(super) async fn get_object_fileinfo(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -410,7 +410,7 @@ impl SetDisks {
|
||||
Ok((fi, parts_metadata, op_online_disks))
|
||||
}
|
||||
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
pub(super) async fn get_object_info_and_quorum(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -605,7 +605,7 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
pub(super) async fn get_object_with_fileinfo<W>(
|
||||
// &self,
|
||||
bucket: &str,
|
||||
@@ -1140,7 +1140,7 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
pub(super) async fn get_object_decode_reader_with_fileinfo(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -1296,7 +1296,7 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
async fn build_codec_streaming_part_reader(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
|
||||
@@ -238,7 +238,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
#[instrument(skip(self, data))]
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "ECStore")]
|
||||
pub(super) async fn handle_put_object_part(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -815,7 +815,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "ECStore")]
|
||||
pub(super) async fn handle_get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -849,7 +849,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, data))]
|
||||
#[hotpath::measure]
|
||||
#[hotpath::measure(impl_type = "ECStore")]
|
||||
pub(super) async fn handle_put_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
const STORE_MULTIPART: &str = include_str!("../src/store/multipart.rs");
|
||||
const STORE_OBJECT: &str = include_str!("../src/store/object.rs");
|
||||
const ERASURE: &str = include_str!("../src/erasure/coding/erasure.rs");
|
||||
const DECODE: &str = include_str!("../src/erasure/coding/decode.rs");
|
||||
const ENCODE: &str = include_str!("../src/erasure/coding/encode.rs");
|
||||
const LOCAL_DISK: &str = include_str!("../src/disk/local.rs");
|
||||
const BITROT: &str = include_str!("../src/erasure/coding/bitrot.rs");
|
||||
const SET_DISK_READ: &str = include_str!("../src/set_disk/read.rs");
|
||||
|
||||
fn assert_measured_as(source: &str, impl_type: &str, function: &str) {
|
||||
let attribute = format!("#[hotpath::measure(impl_type = \"{impl_type}\")]\n");
|
||||
let function_start = format!("fn {function}");
|
||||
let mut remaining = source;
|
||||
|
||||
while let Some(attribute_offset) = remaining.find(&attribute) {
|
||||
let measured_source = &remaining[attribute_offset + attribute.len()..];
|
||||
if measured_source.find(&function_start).is_some_and(|offset| offset < 256) {
|
||||
return;
|
||||
}
|
||||
remaining = measured_source;
|
||||
}
|
||||
|
||||
panic!("missing HotPath impl_type={impl_type} for {function}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inherent_hotpath_measurements_have_cpu_attribution_types() {
|
||||
assert_measured_as(STORE_MULTIPART, "ECStore", "handle_put_object_part");
|
||||
for function in ["handle_get_object_reader", "handle_put_object"] {
|
||||
assert_measured_as(STORE_OBJECT, "ECStore", function);
|
||||
}
|
||||
for function in [
|
||||
"encode_data",
|
||||
"encode_data_owned",
|
||||
"encode_data_bytes_mut",
|
||||
"decode_data",
|
||||
"decode_data_and_parity",
|
||||
] {
|
||||
assert_measured_as(ERASURE, "Erasure", function);
|
||||
}
|
||||
assert_measured_as(DECODE, "ParallelReader", "read");
|
||||
assert_measured_as(DECODE, "Erasure", "decode");
|
||||
for function in [
|
||||
"encode",
|
||||
"encode_batched",
|
||||
"encode_inline_small",
|
||||
"encode_single_block_non_inline",
|
||||
] {
|
||||
assert_measured_as(ENCODE, "Erasure", function);
|
||||
}
|
||||
for function in ["read_metadata_with_dmtime", "read_all_data"] {
|
||||
assert_measured_as(LOCAL_DISK, "LocalDisk", function);
|
||||
}
|
||||
assert_measured_as(BITROT, "BitrotReader", "read");
|
||||
assert!(
|
||||
BITROT.contains("#[hotpath::measure(label = \"BitrotWriter::write\", impl_type = \"BitrotWriter\")]"),
|
||||
"BitrotWriter::write must retain its stable label and CPU impl_type",
|
||||
);
|
||||
for function in [
|
||||
"read_version_optimized",
|
||||
"get_object_fileinfo",
|
||||
"get_object_info_and_quorum",
|
||||
"get_object_with_fileinfo",
|
||||
"get_object_decode_reader_with_fileinfo",
|
||||
"build_codec_streaming_part_reader",
|
||||
] {
|
||||
assert_measured_as(SET_DISK_READ, "SetDisks", function);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn module_hotpath_measurements_remain_untyped() {
|
||||
assert!(
|
||||
BITROT.contains("#[hotpath::measure]\npub async fn bitrot_verify"),
|
||||
"bitrot_verify is a module function and must not claim an impl type",
|
||||
);
|
||||
assert!(
|
||||
SET_DISK_READ.contains("#[hotpath::measure]\nasync fn setup_multipart_part_readers"),
|
||||
"setup_multipart_part_readers is a module function and must not claim an impl type",
|
||||
);
|
||||
}
|
||||
@@ -90,7 +90,13 @@ fn get_policy_plugin_state() -> Arc<RwLock<PolicyPluginState>> {
|
||||
}
|
||||
Ok(_) => PolicyPluginState::Failed,
|
||||
Err(e) => {
|
||||
error!("Error loading OPA configuration err:{}", e);
|
||||
error!(
|
||||
component = "iam",
|
||||
subsystem = "policy_plugin",
|
||||
result = "configuration_load_failed",
|
||||
error_kind = e.kind(),
|
||||
"OPA plugin configuration load failed"
|
||||
);
|
||||
PolicyPluginState::Failed
|
||||
}
|
||||
};
|
||||
|
||||
@@ -2195,18 +2195,22 @@ pub fn record_allocator_memory_observation(backend: &'static str, observation: A
|
||||
}
|
||||
}
|
||||
|
||||
/// Track encoded bytes currently queued between erasure encode and disk writers.
|
||||
/// Track encoded bytes in the queue hand-off between erasure encode and disk writers.
|
||||
///
|
||||
/// This is queue occupancy, not a per-request or process-RSS memory limit. The
|
||||
/// erasure encoder settles it on failed/cancelled sends, receiver drop, and
|
||||
/// consumer hand-off before shard writes begin.
|
||||
#[inline(always)]
|
||||
pub fn add_ec_encode_inflight_bytes(bytes: usize) {
|
||||
let next = EC_ENCODE_INFLIGHT_BYTES.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64;
|
||||
gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
|
||||
gauge_set_cached!("rustfs_ec_encode_inflight_bytes_current", next as f64);
|
||||
}
|
||||
|
||||
/// Remove encoded bytes from the tracked erasure encode in-flight gauge.
|
||||
#[inline(always)]
|
||||
pub fn remove_ec_encode_inflight_bytes(bytes: usize) {
|
||||
let next = saturating_sub_atomic(&EC_ENCODE_INFLIGHT_BYTES, bytes as u64);
|
||||
gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
|
||||
gauge_set_cached!("rustfs_ec_encode_inflight_bytes_current", next as f64);
|
||||
}
|
||||
|
||||
/// Return the current tracked EC encode in-flight bytes.
|
||||
|
||||
@@ -77,6 +77,16 @@ vaultrs = { workspace = true }
|
||||
rustify = { workspace = true }
|
||||
tokio-util = { workspace = true }
|
||||
|
||||
# AWS KMS backend. Credentials come from the standard aws-config provider chain
|
||||
# (environment, shared profile, IMDS/container roles); this crate never handles
|
||||
# AWS credential material itself.
|
||||
aws-config = { workspace = true }
|
||||
aws-sdk-kms = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
# SdkError variants and raw HTTP status are needed to classify AWS failures for
|
||||
# the operation policy's retry decisions.
|
||||
aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
|
||||
aws-smithy-types = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow = { workspace = true }
|
||||
# Debugging recorder for asserting emitted metrics in tests.
|
||||
@@ -86,6 +96,9 @@ tempfile = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
# "net" backs the scripted loopback Vault used by the policy wiring tests.
|
||||
tokio = { workspace = true, features = ["net", "test-util"] }
|
||||
# Replays canned AWS KMS HTTP exchanges so the AWS backend tests stay offline.
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["test-util"] }
|
||||
http = { workspace = true }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
|
||||
+13
-19
@@ -16,8 +16,8 @@
|
||||
|
||||
use crate::config::{
|
||||
BackendConfig, CacheConfig, DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX, DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT, KmsBackend,
|
||||
KmsConfig, LocalConfig, StaticConfig, TlsConfig, VaultAuthMethod, VaultConfig, VaultTransitConfig,
|
||||
allow_immediate_deletion_from_env, redacted_secret, redacted_secret_option,
|
||||
KmsConfig, LocalConfig, StaticConfig, TlsConfig, VaultAuthMethod, VaultConfig, VaultTransitConfig, redacted_secret,
|
||||
redacted_secret_option,
|
||||
};
|
||||
use crate::service_manager::KmsServiceStatus;
|
||||
use crate::types::{KeyMetadata, KeyUsage};
|
||||
@@ -418,6 +418,13 @@ pub enum BackendSummary {
|
||||
/// Configured key identifier
|
||||
key_id: String,
|
||||
},
|
||||
/// AWS KMS backend summary
|
||||
Aws {
|
||||
/// Configured region, when pinned instead of resolved by the AWS chain
|
||||
region: Option<String>,
|
||||
/// Endpoint override, when set for an emulator or private endpoint
|
||||
endpoint_url: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl From<&KmsConfig> for KmsConfigSummary {
|
||||
@@ -467,6 +474,10 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
||||
BackendConfig::Static(static_config) => BackendSummary::Static {
|
||||
key_id: static_config.key_id.clone(),
|
||||
},
|
||||
BackendConfig::Aws(aws_config) => BackendSummary::Aws {
|
||||
region: aws_config.region.clone(),
|
||||
endpoint_url: aws_config.endpoint_url.clone(),
|
||||
},
|
||||
};
|
||||
|
||||
Self {
|
||||
@@ -495,10 +506,6 @@ impl ConfigureLocalKmsRequest {
|
||||
file_permissions: self.file_permissions,
|
||||
}),
|
||||
allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
|
||||
// Read from server configuration, never from the request body: the
|
||||
// gate must mean the same thing whether KMS was configured at
|
||||
// startup or through this endpoint.
|
||||
allow_immediate_deletion: allow_immediate_deletion_from_env(),
|
||||
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
|
||||
retry_attempts: self.retry_attempts.unwrap_or(3),
|
||||
enable_cache: self.enable_cache.unwrap_or(true),
|
||||
@@ -536,10 +543,6 @@ impl ConfigureVaultKmsRequest {
|
||||
},
|
||||
})),
|
||||
allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
|
||||
// Read from server configuration, never from the request body: the
|
||||
// gate must mean the same thing whether KMS was configured at
|
||||
// startup or through this endpoint.
|
||||
allow_immediate_deletion: allow_immediate_deletion_from_env(),
|
||||
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
|
||||
retry_attempts: self.retry_attempts.unwrap_or(3),
|
||||
enable_cache: self.enable_cache.unwrap_or(true),
|
||||
@@ -577,10 +580,6 @@ impl ConfigureVaultTransitKmsRequest {
|
||||
},
|
||||
})),
|
||||
allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
|
||||
// Read from server configuration, never from the request body: the
|
||||
// gate must mean the same thing whether KMS was configured at
|
||||
// startup or through this endpoint.
|
||||
allow_immediate_deletion: allow_immediate_deletion_from_env(),
|
||||
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
|
||||
retry_attempts: self.retry_attempts.unwrap_or(3),
|
||||
enable_cache: self.enable_cache.unwrap_or(true),
|
||||
@@ -604,10 +603,6 @@ impl ConfigureStaticKmsRequest {
|
||||
secret_key: self.secret_key.clone(),
|
||||
}),
|
||||
allow_insecure_dev_defaults: self.allow_insecure_dev_defaults.unwrap_or(false),
|
||||
// Read from server configuration, never from the request body: the
|
||||
// gate must mean the same thing whether KMS was configured at
|
||||
// startup or through this endpoint.
|
||||
allow_immediate_deletion: allow_immediate_deletion_from_env(),
|
||||
timeout: Duration::from_secs(self.timeout_seconds.unwrap_or(30)),
|
||||
retry_attempts: self.retry_attempts.unwrap_or(3),
|
||||
enable_cache: self.enable_cache.unwrap_or(true),
|
||||
@@ -863,7 +858,6 @@ mod tests {
|
||||
tls: None,
|
||||
})),
|
||||
allow_insecure_dev_defaults: true,
|
||||
allow_immediate_deletion: false,
|
||||
timeout: Duration::from_secs(30),
|
||||
retry_attempts: 3,
|
||||
enable_cache: true,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -108,7 +108,6 @@ fn schedule_request(key_id: &str) -> DeleteKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: None,
|
||||
confirm_key_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,10 @@
|
||||
|
||||
//! Local file-based KMS backend implementation
|
||||
|
||||
use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits};
|
||||
use crate::backends::{
|
||||
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits,
|
||||
ensure_tag_keys_are_mutable,
|
||||
};
|
||||
use crate::config::KmsConfig;
|
||||
use crate::config::LocalConfig;
|
||||
use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
|
||||
@@ -1294,6 +1297,32 @@ impl LocalKmsClient {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read-modify-write of a key's mutable metadata under the per-key write
|
||||
/// lock, carrying the existing key material over untouched.
|
||||
///
|
||||
/// `mutate` reports whether it changed anything; an unchanged record is
|
||||
/// never rewritten, so a repeated no-op update neither rewrites the key
|
||||
/// file nor risks a failed commit on an unrelated call.
|
||||
async fn update_key_metadata<F>(&self, key_id: &str, mutate: F) -> Result<()>
|
||||
where
|
||||
F: FnOnce(&mut MasterKeyInfo) -> Result<bool>,
|
||||
{
|
||||
let _write_guard = self.lock_key_for_write(key_id).await;
|
||||
let mut master_key = self.load_master_key(key_id).await?;
|
||||
if !mutate(&mut master_key)? {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Preserve the existing key material (see enable_key): a metadata edit
|
||||
// must never regenerate the master key, or every DEK wrapped by it
|
||||
// becomes undecryptable.
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
debug!(key_id, "Local KMS key metadata updated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test-only lifecycle driver: the product path goes through [`KmsBackend`].
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn schedule_key_deletion(
|
||||
@@ -1386,7 +1415,8 @@ impl LocalKmsBackend {
|
||||
crate::config::BackendConfig::Local(local_config) => local_config.clone(),
|
||||
crate::config::BackendConfig::VaultKv2(_)
|
||||
| crate::config::BackendConfig::VaultTransit(_)
|
||||
| crate::config::BackendConfig::Static(_) => {
|
||||
| crate::config::BackendConfig::Static(_)
|
||||
| crate::config::BackendConfig::Aws(_) => {
|
||||
return Err(KmsError::configuration_error("Expected Local backend configuration"));
|
||||
}
|
||||
};
|
||||
@@ -1411,12 +1441,15 @@ impl KmsBackend for LocalKmsBackend {
|
||||
// Generate key material
|
||||
let key_material = generate_key_material(algorithm)?;
|
||||
|
||||
let master_key = MasterKeyInfo::new_with_description(
|
||||
let mut master_key = MasterKeyInfo::new_with_description(
|
||||
key_id.clone(),
|
||||
algorithm.to_string(),
|
||||
Some("local-kms".to_string()),
|
||||
request.description.clone(),
|
||||
);
|
||||
// Persist the caller's tags: the response below reports them, and
|
||||
// describe_key reads them back out of this record.
|
||||
master_key.metadata = request.tags.clone();
|
||||
|
||||
// Save to disk
|
||||
self.client.save_new_master_key(&master_key, &key_material).await?;
|
||||
@@ -1584,15 +1617,9 @@ impl KmsBackend for LocalKmsBackend {
|
||||
// Schedule for deletion (default 30 days)
|
||||
ensure_key_status_permits(key_id, &master_key.status, StateGatedOperation::ScheduleDeletion)?;
|
||||
|
||||
// Defensive: KmsManager::delete_key is the enforcement point for the
|
||||
// waiting window and rejects out-of-range requests before any
|
||||
// backend runs. This repeats the bound for callers holding a backend
|
||||
// handle directly (tests, maintenance tasks).
|
||||
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
|
||||
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
|
||||
return Err(KmsError::invalid_parameter(format!(
|
||||
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
|
||||
)));
|
||||
let days = request.pending_window_in_days.unwrap_or(30);
|
||||
if !(7..=30).contains(&days) {
|
||||
return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30".to_string()));
|
||||
}
|
||||
|
||||
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
@@ -1690,6 +1717,44 @@ impl KmsBackend for LocalKmsBackend {
|
||||
self.client.disable_key(key_id, None).await
|
||||
}
|
||||
|
||||
async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
|
||||
self.client
|
||||
.update_key_metadata(key_id, |master_key| {
|
||||
if master_key.description.as_deref() == description {
|
||||
return Ok(false);
|
||||
}
|
||||
master_key.description = description.map(str::to_string);
|
||||
Ok(true)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
|
||||
ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
|
||||
self.client
|
||||
.update_key_metadata(key_id, |master_key| {
|
||||
let mut changed = false;
|
||||
for (tag_key, value) in tags {
|
||||
changed |= master_key.metadata.insert(tag_key.clone(), value.clone()).as_ref() != Some(value);
|
||||
}
|
||||
Ok(changed)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
|
||||
ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
|
||||
self.client
|
||||
.update_key_metadata(key_id, |master_key| {
|
||||
let mut changed = false;
|
||||
for tag_key in tag_keys {
|
||||
changed |= master_key.metadata.remove(tag_key).is_some();
|
||||
}
|
||||
Ok(changed)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> Result<bool> {
|
||||
self.client.health_check().await.map(|_| true)
|
||||
}
|
||||
@@ -1702,6 +1767,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
.with_enable_disable(true)
|
||||
.with_schedule_deletion(true)
|
||||
.with_physical_delete(true)
|
||||
.with_update_key_metadata(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
@@ -2612,7 +2678,6 @@ mod tests {
|
||||
key_id: "durable-key".to_string(),
|
||||
pending_window_in_days: None,
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("delete key");
|
||||
@@ -2620,41 +2685,6 @@ mod tests {
|
||||
assert!(fsync_recorder::dir_sync_count(dir) > dirs_before, "delete must fsync the key directory");
|
||||
}
|
||||
|
||||
/// KmsManager::delete_key is the enforcement point for the waiting window;
|
||||
/// this pins the backend's defensive copy of the same bound, which is all
|
||||
/// that stands between a direct backend caller and a one-day window.
|
||||
#[tokio::test]
|
||||
async fn delete_key_refuses_a_window_outside_the_supported_range() {
|
||||
let (client, _temp_dir) = create_test_client().await;
|
||||
let key_id = "window-bounds-key";
|
||||
client.create_key(key_id, "AES_256", None).await.expect("create key");
|
||||
let backend = LocalKmsBackend { client };
|
||||
|
||||
for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
|
||||
let result = backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
pending_window_in_days: Some(days),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(KmsError::InvalidOperation { .. })),
|
||||
"a {days}-day window must be refused, got {result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
let state = backend
|
||||
.describe_key(DescribeKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("describe should succeed")
|
||||
.key_metadata
|
||||
.key_state;
|
||||
assert_eq!(state, KeyState::Enabled, "a refused window must not schedule the key");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn interrupted_update_commit_recovers_to_complete_old_or_new_state() {
|
||||
use durable_file::{CommitStep, failpoint};
|
||||
|
||||
@@ -19,7 +19,9 @@ use crate::types::*;
|
||||
use async_trait::async_trait;
|
||||
use jiff::Zoned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub mod aws;
|
||||
#[cfg(test)]
|
||||
mod contract_tests;
|
||||
pub mod local;
|
||||
@@ -98,6 +100,32 @@ pub(crate) fn ensure_key_status_permits(key_id: &str, status: &KeyStatus, operat
|
||||
ensure_key_state_permits(key_id, &state, operation)
|
||||
}
|
||||
|
||||
/// Tag key that carries the key's identity rather than user metadata.
|
||||
///
|
||||
/// The key-creation path lifts `name` out of the caller's tag map and uses it
|
||||
/// as the key id, so a key's `name` tag and its id are the same string.
|
||||
/// Rewriting or dropping it after creation would leave the key addressable
|
||||
/// under an id its own metadata no longer states. Metadata updates therefore
|
||||
/// reject it; only creation may set it.
|
||||
pub const RESERVED_KEY_NAME_TAG: &str = "name";
|
||||
|
||||
/// Reject a metadata update that would rewrite or remove
|
||||
/// [`RESERVED_KEY_NAME_TAG`].
|
||||
///
|
||||
/// Enforced by every backend that implements tag updates, so no call path —
|
||||
/// including a direct [`KmsBackend`] user that bypasses the manager — can
|
||||
/// detach a key from its identity.
|
||||
pub(crate) fn ensure_tag_keys_are_mutable<'a>(tag_keys: impl IntoIterator<Item = &'a str>) -> Result<()> {
|
||||
for tag_key in tag_keys {
|
||||
if tag_key == RESERVED_KEY_NAME_TAG {
|
||||
return Err(KmsError::invalid_parameter(format!(
|
||||
"Tag '{RESERVED_KEY_NAME_TAG}' identifies the key and cannot be updated or removed"
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Simplified KMS backend interface for manager
|
||||
#[async_trait]
|
||||
pub trait KmsBackend: Send + Sync {
|
||||
@@ -153,6 +181,38 @@ pub trait KmsBackend: Send + Sync {
|
||||
Err(KmsError::unsupported_capability("backend without rotation support", "rotate_key"))
|
||||
}
|
||||
|
||||
/// Replace a key's free-form description; `None` clears it.
|
||||
///
|
||||
/// Backends that advertise [`BackendCapabilities::update_key_metadata`]
|
||||
/// must override this method; the default rejects the operation.
|
||||
async fn update_key_description(&self, _key_id: &str, _description: Option<&str>) -> Result<()> {
|
||||
Err(KmsError::unsupported_capability(
|
||||
"backend without key metadata updates",
|
||||
"update_key_description",
|
||||
))
|
||||
}
|
||||
|
||||
/// Add or overwrite the given tags, leaving every other tag untouched.
|
||||
///
|
||||
/// Implementations must run [`ensure_tag_keys_are_mutable`] before
|
||||
/// persisting anything. Backends that advertise
|
||||
/// [`BackendCapabilities::update_key_metadata`] must override this method;
|
||||
/// the default rejects the operation.
|
||||
async fn tag_key(&self, _key_id: &str, _tags: &HashMap<String, String>) -> Result<()> {
|
||||
Err(KmsError::unsupported_capability("backend without key metadata updates", "tag_key"))
|
||||
}
|
||||
|
||||
/// Remove the given tags.
|
||||
///
|
||||
/// Tags that are not set are ignored, so repeating the call is a no-op
|
||||
/// rather than an error. Implementations must run
|
||||
/// [`ensure_tag_keys_are_mutable`] before persisting anything. Backends
|
||||
/// that advertise [`BackendCapabilities::update_key_metadata`] must
|
||||
/// override this method; the default rejects the operation.
|
||||
async fn untag_key(&self, _key_id: &str, _tag_keys: &[String]) -> Result<()> {
|
||||
Err(KmsError::unsupported_capability("backend without key metadata updates", "untag_key"))
|
||||
}
|
||||
|
||||
/// Health check
|
||||
async fn health_check(&self) -> Result<bool>;
|
||||
|
||||
@@ -222,6 +282,8 @@ pub struct BackendCapabilities {
|
||||
pub versioning: bool,
|
||||
/// Irreversible physical deletion of key material
|
||||
pub physical_delete: bool,
|
||||
/// Updating a key's description and tags after creation
|
||||
pub update_key_metadata: bool,
|
||||
}
|
||||
|
||||
impl BackendCapabilities {
|
||||
@@ -238,6 +300,7 @@ impl BackendCapabilities {
|
||||
schedule_deletion: false,
|
||||
versioning: false,
|
||||
physical_delete: false,
|
||||
update_key_metadata: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -288,6 +351,12 @@ impl BackendCapabilities {
|
||||
self.physical_delete = physical_delete;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether description and tag updates are supported
|
||||
pub const fn with_update_key_metadata(mut self, update_key_metadata: bool) -> Self {
|
||||
self.update_key_metadata = update_key_metadata;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for BackendCapabilities {
|
||||
@@ -366,6 +435,7 @@ mod tests {
|
||||
assert!(!capabilities.schedule_deletion);
|
||||
assert!(!capabilities.versioning);
|
||||
assert!(!capabilities.physical_delete);
|
||||
assert!(!capabilities.update_key_metadata);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -374,6 +444,12 @@ mod tests {
|
||||
("enable_key", MinimalBackend.enable_key("any-key").await),
|
||||
("disable_key", MinimalBackend.disable_key("any-key").await),
|
||||
("rotate_key", MinimalBackend.rotate_key("any-key").await),
|
||||
(
|
||||
"update_key_description",
|
||||
MinimalBackend.update_key_description("any-key", Some("new")).await,
|
||||
),
|
||||
("tag_key", MinimalBackend.tag_key("any-key", &HashMap::new()).await),
|
||||
("untag_key", MinimalBackend.untag_key("any-key", &[]).await),
|
||||
] {
|
||||
let error = result.expect_err("backends must opt in to lifecycle operations by overriding them");
|
||||
assert!(
|
||||
@@ -395,6 +471,20 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_tag_is_rejected_by_metadata_updates() {
|
||||
let error = ensure_tag_keys_are_mutable([RESERVED_KEY_NAME_TAG])
|
||||
.expect_err("the identity tag must not be writable through a metadata update");
|
||||
assert!(
|
||||
matches!(&error, KmsError::InvalidOperation { message } if message.contains(RESERVED_KEY_NAME_TAG)),
|
||||
"expected a typed rejection naming the tag, got {error:?}"
|
||||
);
|
||||
|
||||
// Ordinary tags — including ones that merely contain the reserved name
|
||||
// — stay writable.
|
||||
ensure_tag_keys_are_mutable(["team", "nickname", "Name"]).expect("ordinary tags must remain writable");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_backend_capabilities_golden() {
|
||||
let temp_dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
---
|
||||
source: crates/kms/src/backends/aws.rs
|
||||
expression: capabilities_snapshot(backend.capabilities())
|
||||
---
|
||||
{
|
||||
"decrypt": true,
|
||||
"enable_disable": true,
|
||||
"encrypt": true,
|
||||
"generate_data_key": true,
|
||||
"physical_delete": false,
|
||||
"rotate": true,
|
||||
"schedule_deletion": true,
|
||||
"update_key_metadata": false,
|
||||
"versioning": true
|
||||
}
|
||||
+1
@@ -10,5 +10,6 @@ expression: capabilities_snapshot(backend.capabilities())
|
||||
"physical_delete": true,
|
||||
"rotate": false,
|
||||
"schedule_deletion": true,
|
||||
"update_key_metadata": true,
|
||||
"versioning": false
|
||||
}
|
||||
|
||||
+1
@@ -10,5 +10,6 @@ expression: capabilities_snapshot(backend.capabilities())
|
||||
"physical_delete": false,
|
||||
"rotate": false,
|
||||
"schedule_deletion": false,
|
||||
"update_key_metadata": false,
|
||||
"versioning": false
|
||||
}
|
||||
|
||||
+1
@@ -10,5 +10,6 @@ expression: capabilities_snapshot(backend.capabilities())
|
||||
"physical_delete": true,
|
||||
"rotate": true,
|
||||
"schedule_deletion": true,
|
||||
"update_key_metadata": true,
|
||||
"versioning": true
|
||||
}
|
||||
|
||||
+1
@@ -10,5 +10,6 @@ expression: capabilities_snapshot(backend.capabilities())
|
||||
"physical_delete": true,
|
||||
"rotate": true,
|
||||
"schedule_deletion": true,
|
||||
"update_key_metadata": true,
|
||||
"versioning": true
|
||||
}
|
||||
|
||||
@@ -431,6 +431,32 @@ mod tests {
|
||||
(backend, key_id, key)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_updates_report_the_capability_gap() {
|
||||
let (backend, key_id, _key) = create_test_backend().await;
|
||||
assert!(
|
||||
!backend.capabilities().update_key_metadata,
|
||||
"Static KMS owns no mutable key record, so it must not advertise metadata updates"
|
||||
);
|
||||
|
||||
for (operation, result) in [
|
||||
("update_key_description", backend.update_key_description(&key_id, Some("new")).await),
|
||||
(
|
||||
"tag_key",
|
||||
backend
|
||||
.tag_key(&key_id, &HashMap::from([("team".to_string(), "storage".to_string())]))
|
||||
.await,
|
||||
),
|
||||
("untag_key", backend.untag_key(&key_id, &["team".to_string()]).await),
|
||||
] {
|
||||
let error = result.expect_err("Static KMS is read-only: metadata updates must be rejected");
|
||||
assert!(
|
||||
matches!(error, KmsError::UnsupportedCapability { .. }),
|
||||
"expected UnsupportedCapability for {operation}, got {error:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_generate_and_decrypt_data_key() {
|
||||
let (backend, key_id, _key) = create_test_backend().await;
|
||||
@@ -657,7 +683,6 @@ mod tests {
|
||||
key_id: key_id.clone(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: None,
|
||||
confirm_key_id: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -20,6 +20,7 @@ use crate::backends::vault_credentials::{
|
||||
};
|
||||
use crate::backends::{
|
||||
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits, ensure_key_status_permits,
|
||||
ensure_tag_keys_are_mutable,
|
||||
};
|
||||
use crate::config::{KmsConfig, VaultConfig};
|
||||
use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
|
||||
@@ -1020,6 +1021,67 @@ impl VaultKmsClient {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Replace the key's description; `None` clears it.
|
||||
///
|
||||
/// The write goes through the check-and-set read-modify-write loop, so a
|
||||
/// rotation or state transition landing in between is carried over instead
|
||||
/// of clobbered. A description that already matches is not rewritten.
|
||||
pub(crate) async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
|
||||
self.update_key_data_with_cas(key_id, |key_data| {
|
||||
if key_data.description.as_deref() == description {
|
||||
return Ok(CasMutation::Skip(()));
|
||||
}
|
||||
key_data.description = description.map(str::to_string);
|
||||
Ok(CasMutation::Write(()))
|
||||
})
|
||||
.await?;
|
||||
|
||||
debug!(key_id, "Vault KMS key description updated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Add or overwrite tags, leaving every other tag untouched.
|
||||
pub(crate) async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
|
||||
ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
|
||||
|
||||
self.update_key_data_with_cas(key_id, |key_data| {
|
||||
let mut changed = false;
|
||||
for (tag_key, value) in tags {
|
||||
changed |= key_data.tags.insert(tag_key.clone(), value.clone()).as_ref() != Some(value);
|
||||
}
|
||||
Ok(if changed {
|
||||
CasMutation::Write(())
|
||||
} else {
|
||||
CasMutation::Skip(())
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
|
||||
debug!(key_id, "Vault KMS key tags updated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove tags; tags that are not set are ignored.
|
||||
pub(crate) async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
|
||||
ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
|
||||
|
||||
self.update_key_data_with_cas(key_id, |key_data| {
|
||||
let mut changed = false;
|
||||
for tag_key in tag_keys {
|
||||
changed |= key_data.tags.remove(tag_key).is_some();
|
||||
}
|
||||
Ok(if changed {
|
||||
CasMutation::Write(())
|
||||
} else {
|
||||
CasMutation::Skip(())
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
|
||||
debug!(key_id, "Vault KMS key tags removed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rotate the master key while keeping every historical version decryptable.
|
||||
///
|
||||
/// Commit protocol (all writes check-and-set, in this order):
|
||||
@@ -1163,7 +1225,8 @@ impl VaultKmsBackend {
|
||||
crate::config::BackendConfig::VaultKv2(vault_config) => (**vault_config).clone(),
|
||||
crate::config::BackendConfig::Local(_)
|
||||
| crate::config::BackendConfig::VaultTransit(_)
|
||||
| crate::config::BackendConfig::Static(_) => {
|
||||
| crate::config::BackendConfig::Static(_)
|
||||
| crate::config::BackendConfig::Aws(_) => {
|
||||
return Err(KmsError::configuration_error("Expected Vault KV2 backend configuration"));
|
||||
}
|
||||
};
|
||||
@@ -1364,15 +1427,11 @@ impl KmsBackend for VaultKmsBackend {
|
||||
// Schedule for deletion (default 30 days)
|
||||
ensure_key_state_permits(key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
|
||||
|
||||
// Defensive: KmsManager::delete_key is the enforcement point for the
|
||||
// waiting window and rejects out-of-range requests before any
|
||||
// backend runs. This repeats the bound for callers holding a backend
|
||||
// handle directly (tests, maintenance tasks).
|
||||
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
|
||||
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
|
||||
return Err(crate::error::KmsError::invalid_parameter(format!(
|
||||
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
|
||||
)));
|
||||
let days = request.pending_window_in_days.unwrap_or(30);
|
||||
if !(7..=30).contains(&days) {
|
||||
return Err(crate::error::KmsError::invalid_parameter(
|
||||
"pending_window_in_days must be between 7 and 30".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let deletion_date = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
@@ -1450,6 +1509,18 @@ impl KmsBackend for VaultKmsBackend {
|
||||
self.client.rotate_key(key_id, None).await.map(|_| ())
|
||||
}
|
||||
|
||||
async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
|
||||
self.client.update_key_description(key_id, description).await
|
||||
}
|
||||
|
||||
async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
|
||||
self.client.tag_key(key_id, tags).await
|
||||
}
|
||||
|
||||
async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
|
||||
self.client.untag_key(key_id, tag_keys).await
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> Result<bool> {
|
||||
self.client.health_check().await.map(|_| true)
|
||||
}
|
||||
@@ -1465,6 +1536,7 @@ impl KmsBackend for VaultKmsBackend {
|
||||
.with_schedule_deletion(true)
|
||||
.with_versioning(true)
|
||||
.with_physical_delete(true)
|
||||
.with_update_key_metadata(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
@@ -2202,7 +2274,6 @@ mod tests {
|
||||
key_id: key_id.clone(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: Some(false),
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("schedule delete");
|
||||
@@ -2706,7 +2777,6 @@ mod tests {
|
||||
key_id: "wired-key".to_string(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: Some(false),
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("the schedule must retry past the lost race and commit");
|
||||
@@ -2723,45 +2793,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// KmsManager::delete_key is the enforcement point for the waiting window;
|
||||
/// this pins the backend's defensive copy of the same bound, which is all
|
||||
/// that stands between a direct backend caller and a one-day window.
|
||||
#[tokio::test]
|
||||
async fn wired_schedule_deletion_refuses_a_window_outside_the_supported_range() {
|
||||
for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// describe_key: key info plus stored metadata.
|
||||
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
|
||||
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
|
||||
])
|
||||
.await;
|
||||
let config = KmsConfig::vault(
|
||||
url::Url::parse(&vault.address).expect("scripted vault address should parse"),
|
||||
"scripted-token".to_string(),
|
||||
)
|
||||
.with_insecure_development_defaults();
|
||||
let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build");
|
||||
|
||||
let result = backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "wired-key".to_string(),
|
||||
pending_window_in_days: Some(days),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(KmsError::InvalidOperation { .. })),
|
||||
"a {days}-day window must be refused, got {result:?}"
|
||||
);
|
||||
|
||||
let requests = vault.requests();
|
||||
assert!(
|
||||
!requests.iter().any(|line| line.starts_with("POST ")),
|
||||
"a refused window must not write anything: {requests:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Conflict semantics are re-read *and* re-gate: when the re-read after a
|
||||
/// lost race shows the key was concurrently scheduled for deletion, the
|
||||
/// state gate rejects the retry instead of blindly re-applying it.
|
||||
@@ -2794,7 +2825,6 @@ mod tests {
|
||||
key_id: "wired-key".to_string(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: Some(false),
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect_err("the retry must re-run the state gate against the fresh record");
|
||||
@@ -2890,6 +2920,63 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Tag updates are check-and-set read-modify-writes over the live record,
|
||||
/// never blind overwrites: they preserve the material and the tags they did
|
||||
/// not address.
|
||||
#[tokio::test]
|
||||
async fn wired_tag_key_writeback_is_check_and_set() {
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.tags = HashMap::from([("name".to_string(), "wired-key".to_string())]);
|
||||
let (vault, client) = scripted_client(vec![
|
||||
ScriptedResponse::ok(kv2_metadata_read_data(1)),
|
||||
ScriptedResponse::ok(kv2_read_data(&key_data)),
|
||||
ScriptedResponse::ok(kv2_write_ack()),
|
||||
])
|
||||
.await;
|
||||
|
||||
client
|
||||
.tag_key("wired-key", &HashMap::from([("team".to_string(), "storage".to_string())]))
|
||||
.await
|
||||
.expect("tagging must succeed");
|
||||
|
||||
let bodies = vault.request_bodies();
|
||||
let writeback = parse_write_body(&bodies[2]);
|
||||
assert_eq!(writeback["options"]["cas"], serde_json::json!(1), "{writeback}");
|
||||
assert_eq!(writeback["data"]["tags"]["team"], serde_json::json!("storage"), "{writeback}");
|
||||
assert_eq!(
|
||||
writeback["data"]["tags"]["name"],
|
||||
serde_json::json!("wired-key"),
|
||||
"a tag update must not drop tags it did not address: {writeback}"
|
||||
);
|
||||
assert_eq!(
|
||||
writeback["data"]["encrypted_key_material"],
|
||||
serde_json::json!(healthy_key_data().encrypted_key_material),
|
||||
"the write-back must preserve the material of the freshly read record: {writeback}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Rejecting the identity tag happens before any Vault call, so a rejected
|
||||
/// request cannot leave a partial write behind.
|
||||
#[tokio::test]
|
||||
async fn wired_identity_tag_update_is_rejected_before_any_vault_call() {
|
||||
let (vault, client) = scripted_client(Vec::new()).await;
|
||||
|
||||
for result in [
|
||||
client
|
||||
.tag_key("wired-key", &HashMap::from([("name".to_string(), "other".to_string())]))
|
||||
.await,
|
||||
client.untag_key("wired-key", &["name".to_string()]).await,
|
||||
] {
|
||||
let error = result.expect_err("the identity tag must not be writable");
|
||||
assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}");
|
||||
}
|
||||
assert!(
|
||||
vault.request_bodies().is_empty(),
|
||||
"a rejected metadata update must not reach Vault: {:?}",
|
||||
vault.request_bodies()
|
||||
);
|
||||
}
|
||||
|
||||
/// A version record above the current pointer means the top-level record
|
||||
/// regressed (a lost update rolled back a committed rotation). Resolving
|
||||
/// material through such a record must fail closed instead of quietly
|
||||
|
||||
@@ -18,7 +18,10 @@ use crate::backends::vault_credentials::{
|
||||
CredentialTaskHandle, VaultClientHandle, VaultConnectionSettings, VaultCredentialPolicy, VaultCredentialProvider,
|
||||
token_source_for,
|
||||
};
|
||||
use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits};
|
||||
use crate::backends::{
|
||||
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits,
|
||||
ensure_tag_keys_are_mutable,
|
||||
};
|
||||
use crate::config::{KmsConfig, VaultTransitConfig};
|
||||
use crate::encryption::{DataKeyEnvelope, generate_key_material};
|
||||
use crate::error::{KmsError, Result};
|
||||
@@ -908,6 +911,46 @@ impl VaultTransitKmsClient {
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Replace the key's description; `None` clears it.
|
||||
///
|
||||
/// Metadata edits carry no state gate: they neither use nor invalidate key
|
||||
/// material, so they stay available for whatever lifecycle state the key
|
||||
/// is in.
|
||||
pub(crate) async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
|
||||
self.mutate_key_metadata(key_id, |metadata| {
|
||||
metadata.description = description.map(str::to_string);
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Add or overwrite tags, leaving every other tag untouched.
|
||||
pub(crate) async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
|
||||
ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
|
||||
self.mutate_key_metadata(key_id, |metadata| {
|
||||
metadata
|
||||
.tags
|
||||
.extend(tags.iter().map(|(key, value)| (key.clone(), value.clone())));
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Remove tags; tags that are not set are ignored.
|
||||
pub(crate) async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
|
||||
ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
|
||||
self.mutate_key_metadata(key_id, |metadata| {
|
||||
for tag_key in tag_keys {
|
||||
metadata.tags.remove(tag_key);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Test-only lifecycle driver: the product path goes through [`KmsBackend`].
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn schedule_key_deletion(
|
||||
@@ -1012,7 +1055,9 @@ impl VaultTransitKmsBackend {
|
||||
metadata_key_prefix: vault_config.key_path_prefix.clone(),
|
||||
tls: vault_config.tls.clone(),
|
||||
},
|
||||
crate::config::BackendConfig::Local(_) | crate::config::BackendConfig::Static(_) => {
|
||||
crate::config::BackendConfig::Local(_)
|
||||
| crate::config::BackendConfig::Static(_)
|
||||
| crate::config::BackendConfig::Aws(_) => {
|
||||
return Err(KmsError::configuration_error("Expected Vault Transit backend configuration"));
|
||||
}
|
||||
};
|
||||
@@ -1174,15 +1219,9 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
} else {
|
||||
ensure_key_state_permits(&key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
|
||||
|
||||
// Defensive: KmsManager::delete_key is the enforcement point for the
|
||||
// waiting window and rejects out-of-range requests before any
|
||||
// backend runs. This repeats the bound for callers holding a backend
|
||||
// handle directly (tests, maintenance tasks).
|
||||
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
|
||||
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
|
||||
return Err(KmsError::invalid_parameter(format!(
|
||||
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
|
||||
)));
|
||||
let days = request.pending_window_in_days.unwrap_or(30);
|
||||
if !(7..=30).contains(&days) {
|
||||
return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30"));
|
||||
}
|
||||
|
||||
let scheduled = Zoned::now() + Duration::from_secs(days as u64 * 86400);
|
||||
@@ -1241,6 +1280,18 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
self.client.rotate_key(key_id, None).await.map(|_| ())
|
||||
}
|
||||
|
||||
async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
|
||||
self.client.update_key_description(key_id, description).await
|
||||
}
|
||||
|
||||
async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
|
||||
self.client.tag_key(key_id, tags).await
|
||||
}
|
||||
|
||||
async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
|
||||
self.client.untag_key(key_id, tag_keys).await
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> Result<bool> {
|
||||
self.client.health_check().await.map(|_| true)
|
||||
}
|
||||
@@ -1255,6 +1306,7 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
.with_schedule_deletion(true)
|
||||
.with_versioning(true)
|
||||
.with_physical_delete(true)
|
||||
.with_update_key_metadata(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
@@ -1767,48 +1819,6 @@ mod tests {
|
||||
assert_eq!(requests[6], "POST /v1/transit/keys/wired-key/rotate", "{requests:?}");
|
||||
}
|
||||
|
||||
/// KmsManager::delete_key is the enforcement point for the waiting window;
|
||||
/// this pins the backend's defensive copy of the same bound, which is all
|
||||
/// that stands between a direct backend caller and a one-day window.
|
||||
#[tokio::test]
|
||||
async fn wired_backend_delete_refuses_a_window_outside_the_supported_range() {
|
||||
for days in [MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
|
||||
let metadata = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// The state gate reads the transit key, then its metadata record.
|
||||
ScriptedResponse::ok(transit_key_read_data("wired-key")),
|
||||
ScriptedResponse::ok(metadata_read_data(&metadata)),
|
||||
])
|
||||
.await;
|
||||
let config = KmsConfig::vault_transit(
|
||||
url::Url::parse(&vault.address).expect("scripted vault address should parse"),
|
||||
"scripted-token".to_string(),
|
||||
)
|
||||
.with_insecure_development_defaults();
|
||||
let backend = VaultTransitKmsBackend::new(config)
|
||||
.await
|
||||
.expect("vault transit backend should build");
|
||||
|
||||
let result = backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "wired-key".to_string(),
|
||||
pending_window_in_days: Some(days),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(KmsError::InvalidOperation { .. })),
|
||||
"a {days}-day window must be refused, got {result:?}"
|
||||
);
|
||||
|
||||
let requests = vault.requests();
|
||||
assert!(
|
||||
!requests.iter().any(|line| line.starts_with("POST ")),
|
||||
"a refused window must not write anything: {requests:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// KV2 secret-metadata read payload (`kv2::read_metadata`) pinning the
|
||||
/// current secret version used as the check-and-set base.
|
||||
fn kv2_metadata_read_data(current_version: u64) -> serde_json::Value {
|
||||
|
||||
+118
-68
@@ -24,7 +24,6 @@ use std::time::Duration;
|
||||
use url::Url;
|
||||
|
||||
pub const ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS: &str = "RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS";
|
||||
pub const ENV_KMS_ALLOW_IMMEDIATE_DELETION: &str = "RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION";
|
||||
pub const ENV_KMS_VAULT_SKIP_TLS_VERIFY: &str = "RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY";
|
||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_KV_MOUNT";
|
||||
pub const ENV_KMS_VAULT_TRANSIT_METADATA_PREFIX: &str = "RUSTFS_KMS_VAULT_TRANSIT_METADATA_PREFIX";
|
||||
@@ -35,6 +34,8 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR
|
||||
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
|
||||
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
|
||||
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
|
||||
pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
|
||||
pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
|
||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
|
||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
|
||||
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
|
||||
@@ -129,6 +130,10 @@ pub enum KmsBackend {
|
||||
/// Static single-key backend that derives DEKs from a pre-configured key
|
||||
#[serde(rename = "Static")]
|
||||
Static,
|
||||
/// AWS KMS backend: AWS is the cryptographic source of truth and owns key
|
||||
/// state, versioning, and the deletion window.
|
||||
#[serde(rename = "AWS", alias = "AwsKms")]
|
||||
Aws,
|
||||
}
|
||||
|
||||
impl KmsBackend {
|
||||
@@ -142,6 +147,7 @@ impl KmsBackend {
|
||||
KmsBackend::VaultTransit => "vault-transit",
|
||||
KmsBackend::Local => "local",
|
||||
KmsBackend::Static => "static",
|
||||
KmsBackend::Aws => "aws",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -158,24 +164,6 @@ pub struct KmsConfig {
|
||||
/// Allow development-only insecure defaults such as plaintext local keys or HTTP Vault.
|
||||
#[serde(default)]
|
||||
pub allow_insecure_dev_defaults: bool,
|
||||
/// Allow `DeleteKey` requests to skip the pending-deletion waiting window and
|
||||
/// destroy key material right away.
|
||||
///
|
||||
/// Off by default: an immediate deletion is unrecoverable and takes every
|
||||
/// object encrypted under the key with it, so the waiting window (plus
|
||||
/// `CancelKeyDeletion`) is the only recovery path. Operators who genuinely
|
||||
/// need immediate deletion — throwaway test clusters, key material that was
|
||||
/// never used — must turn it on through server configuration
|
||||
/// ([`ENV_KMS_ALLOW_IMMEDIATE_DELETION`]); the request must still echo the
|
||||
/// key id back for confirmation.
|
||||
///
|
||||
/// Not part of the serialized configuration, and not settable through the
|
||||
/// admin configure API. It is per-server operator state that has to be
|
||||
/// re-stated to survive a restart: persisting it would carry one operator's
|
||||
/// one-time enablement into the cluster-wide config that every node reloads,
|
||||
/// long after the deletion it was turned on for.
|
||||
#[serde(skip)]
|
||||
pub allow_immediate_deletion: bool,
|
||||
/// Timeout for a single backend attempt.
|
||||
///
|
||||
/// This bounds one outbound request, not the whole operation: the operation
|
||||
@@ -199,7 +187,6 @@ impl Default for KmsConfig {
|
||||
default_key_id: None,
|
||||
backend_config: BackendConfig::default(),
|
||||
allow_insecure_dev_defaults: false,
|
||||
allow_immediate_deletion: false,
|
||||
timeout: Duration::from_secs(30),
|
||||
retry_attempts: 3,
|
||||
enable_cache: true,
|
||||
@@ -220,6 +207,9 @@ pub enum BackendConfig {
|
||||
VaultTransit(Box<VaultTransitConfig>),
|
||||
/// Static single-key backend configuration
|
||||
Static(StaticConfig),
|
||||
/// AWS KMS backend configuration
|
||||
#[serde(rename = "AWS", alias = "AwsKms")]
|
||||
Aws(Box<AwsKmsConfig>),
|
||||
}
|
||||
|
||||
impl Default for BackendConfig {
|
||||
@@ -235,6 +225,7 @@ impl fmt::Debug for BackendConfig {
|
||||
Self::VaultKv2(config) => f.debug_tuple("VaultKv2").field(config).finish(),
|
||||
Self::VaultTransit(config) => f.debug_tuple("VaultTransit").field(config).finish(),
|
||||
Self::Static(config) => f.debug_tuple("Static").field(config).finish(),
|
||||
Self::Aws(config) => f.debug_tuple("Aws").field(config).finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -560,6 +551,25 @@ impl Default for CacheConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// AWS KMS backend configuration.
|
||||
///
|
||||
/// Deliberately holds no credential material: the backend resolves credentials
|
||||
/// through the standard `aws-config` provider chain (environment, shared
|
||||
/// profile, container/IMDS role), so RustFS never stores, persists, or redacts
|
||||
/// AWS secrets of its own.
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct AwsKmsConfig {
|
||||
/// AWS region hosting the KMS keys. When unset, the region is resolved by
|
||||
/// the standard chain (`AWS_REGION`, profile, IMDS).
|
||||
#[serde(default)]
|
||||
pub region: Option<String>,
|
||||
/// Override for the KMS endpoint, for local emulators and private
|
||||
/// endpoints. Unset in production, where the SDK derives the regional
|
||||
/// endpoint.
|
||||
#[serde(default)]
|
||||
pub endpoint_url: Option<String>,
|
||||
}
|
||||
|
||||
impl KmsConfig {
|
||||
/// Create a new KMS configuration for local backend (for development and testing only)
|
||||
pub fn local(key_dir: PathBuf) -> Self {
|
||||
@@ -669,6 +679,29 @@ impl KmsConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new KMS configuration for the AWS KMS backend.
|
||||
///
|
||||
/// Credentials are resolved by the standard `aws-config` provider chain;
|
||||
/// only the region is configured here.
|
||||
pub fn aws(region: Option<String>) -> Self {
|
||||
Self {
|
||||
backend: KmsBackend::Aws,
|
||||
backend_config: BackendConfig::Aws(Box::new(AwsKmsConfig {
|
||||
region,
|
||||
endpoint_url: None,
|
||||
})),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the AWS configuration if backend is AWS KMS
|
||||
pub fn aws_kms_config(&self) -> Option<&AwsKmsConfig> {
|
||||
match &self.backend_config {
|
||||
BackendConfig::Aws(config) => Some(config),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set default key ID
|
||||
pub fn with_default_key(mut self, key_id: String) -> Self {
|
||||
self.default_key_id = Some(key_id);
|
||||
@@ -681,12 +714,6 @@ impl KmsConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Explicitly allow deletions that bypass the pending-deletion waiting window.
|
||||
pub fn with_immediate_deletion_allowed(mut self) -> Self {
|
||||
self.allow_immediate_deletion = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set operation timeout
|
||||
pub fn with_timeout(mut self, timeout: Duration) -> Self {
|
||||
self.timeout = timeout;
|
||||
@@ -831,6 +858,25 @@ impl KmsConfig {
|
||||
// Validate that the key can be decoded (right length, valid base64)
|
||||
config.decode_key()?;
|
||||
}
|
||||
BackendConfig::Aws(config) => {
|
||||
if let Some(region) = &config.region
|
||||
&& region.is_empty()
|
||||
{
|
||||
return Err(KmsError::configuration_error("AWS KMS region cannot be empty when set"));
|
||||
}
|
||||
|
||||
if let Some(endpoint) = &config.endpoint_url {
|
||||
if !endpoint.starts_with("http://") && !endpoint.starts_with("https://") {
|
||||
return Err(KmsError::configuration_error("AWS KMS endpoint URL must use http or https scheme"));
|
||||
}
|
||||
// A plaintext endpoint override exposes every KMS request,
|
||||
// including plaintext data keys, so it stays gated on the
|
||||
// explicit development opt-in.
|
||||
if endpoint.starts_with("http://") && !self.allow_insecure_dev_defaults {
|
||||
return Err(development_default_error("AWS KMS endpoint URL must use https"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate cache configuration
|
||||
@@ -852,6 +898,7 @@ impl KmsConfig {
|
||||
"vault" | "vault-kv2" | "vault_kv2" => KmsBackend::VaultKv2,
|
||||
"vault-transit" | "vault_transit" => KmsBackend::VaultTransit,
|
||||
"static" => KmsBackend::Static,
|
||||
"aws" | "aws-kms" | "aws_kms" => KmsBackend::Aws,
|
||||
_ => return Err(KmsError::configuration_error(format!("Unknown KMS backend: {backend_type}"))),
|
||||
};
|
||||
}
|
||||
@@ -880,7 +927,6 @@ impl KmsConfig {
|
||||
config.enable_cache = get_env_bool("RUSTFS_KMS_ENABLE_CACHE", config.enable_cache);
|
||||
config.allow_insecure_dev_defaults =
|
||||
get_env_bool(ENV_KMS_ALLOW_INSECURE_DEV_DEFAULTS, config.allow_insecure_dev_defaults);
|
||||
config.allow_immediate_deletion = get_env_bool(ENV_KMS_ALLOW_IMMEDIATE_DELETION, config.allow_immediate_deletion);
|
||||
|
||||
// Backend-specific configuration
|
||||
match config.backend {
|
||||
@@ -981,6 +1027,14 @@ impl KmsConfig {
|
||||
});
|
||||
config.default_key_id = Some(key_id);
|
||||
}
|
||||
KmsBackend::Aws => {
|
||||
// Only non-credential settings are read here; access keys,
|
||||
// profiles, and role assumption stay with the aws-config chain.
|
||||
config.backend_config = BackendConfig::Aws(Box::new(AwsKmsConfig {
|
||||
region: get_env_opt_str(ENV_KMS_AWS_REGION),
|
||||
endpoint_url: get_env_opt_str(ENV_KMS_AWS_ENDPOINT_URL),
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
config.validate()?;
|
||||
@@ -988,15 +1042,6 @@ impl KmsConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the immediate-deletion gate from the environment.
|
||||
///
|
||||
/// Callers that assemble a [`KmsConfig`] field by field instead of going
|
||||
/// through [`KmsConfig::from_env`] use this, so the gate keeps one name, one
|
||||
/// default, and one place to look it up.
|
||||
pub fn allow_immediate_deletion_from_env() -> bool {
|
||||
get_env_bool(ENV_KMS_ALLOW_IMMEDIATE_DELETION, false)
|
||||
}
|
||||
|
||||
fn vault_tls_config(skip_tls_verify: bool) -> Option<TlsConfig> {
|
||||
skip_tls_verify.then_some(TlsConfig {
|
||||
ca_cert_path: None,
|
||||
@@ -1591,6 +1636,43 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// The AWS backend reads only non-credential settings from the
|
||||
/// environment; access keys, profiles, and role assumption stay with the
|
||||
/// aws-config provider chain.
|
||||
#[test]
|
||||
fn test_from_env_selects_aws_backend_without_credentials() {
|
||||
with_vars(
|
||||
vec![
|
||||
("RUSTFS_KMS_BACKEND", Some("aws")),
|
||||
(ENV_KMS_AWS_REGION, Some("eu-central-1")),
|
||||
(ENV_KMS_AWS_ENDPOINT_URL, None::<&str>),
|
||||
],
|
||||
|| {
|
||||
let config = KmsConfig::from_env().expect("kms config should load from env");
|
||||
assert_eq!(config.backend, KmsBackend::Aws);
|
||||
let aws = config.aws_kms_config().expect("aws backend config");
|
||||
assert_eq!(aws.region.as_deref(), Some("eu-central-1"));
|
||||
assert_eq!(aws.endpoint_url, None);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// A plaintext endpoint override exposes every KMS request, including the
|
||||
/// plaintext data keys, so it stays behind the explicit development opt-in.
|
||||
#[test]
|
||||
fn test_from_env_rejects_plaintext_aws_endpoint() {
|
||||
with_vars(
|
||||
vec![
|
||||
("RUSTFS_KMS_BACKEND", Some("aws")),
|
||||
(ENV_KMS_AWS_REGION, Some("us-east-1")),
|
||||
(ENV_KMS_AWS_ENDPOINT_URL, Some("http://localhost:4566")),
|
||||
],
|
||||
|| {
|
||||
KmsConfig::from_env().expect_err("a plaintext AWS endpoint must be rejected by default");
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_env_approle_requires_secret_id_or_file() {
|
||||
with_vars(
|
||||
@@ -1720,38 +1802,6 @@ mod tests {
|
||||
assert_eq!(refresh_safety_window_secs, None);
|
||||
}
|
||||
|
||||
/// The gate lives in server configuration only: it never rides along in a
|
||||
/// serialized config, and a stored config that claims it must not be
|
||||
/// believed. Otherwise one operator's one-time enablement would reach every
|
||||
/// node that later reloads that config.
|
||||
#[test]
|
||||
fn immediate_deletion_gate_is_server_local_and_never_persisted() {
|
||||
let persisted =
|
||||
serde_json::to_value(KmsConfig::default().with_immediate_deletion_allowed()).expect("kms config should serialize");
|
||||
assert!(
|
||||
persisted.get("allow_immediate_deletion").is_none(),
|
||||
"the gate must not be written into a persisted config: {persisted}"
|
||||
);
|
||||
|
||||
let mut forged = persisted;
|
||||
forged
|
||||
.as_object_mut()
|
||||
.expect("a persisted config must be a JSON object")
|
||||
.insert("allow_immediate_deletion".to_string(), serde_json::json!(true));
|
||||
let restored: KmsConfig = serde_json::from_value(forged).expect("an unknown gate field must not break loading");
|
||||
assert!(!restored.allow_immediate_deletion, "a stored gate must fail closed");
|
||||
|
||||
with_vars(vec![(ENV_KMS_ALLOW_IMMEDIATE_DELETION, Some("true"))], || {
|
||||
assert!(
|
||||
allow_immediate_deletion_from_env(),
|
||||
"the gate must be reachable from server configuration"
|
||||
);
|
||||
});
|
||||
with_vars(vec![(ENV_KMS_ALLOW_IMMEDIATE_DELETION, None::<&str>)], || {
|
||||
assert!(!allow_immediate_deletion_from_env());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_rejects_incomplete_approle() {
|
||||
let mut config = KmsConfig::vault_approle(
|
||||
|
||||
@@ -376,7 +376,6 @@ mod tests {
|
||||
key_id: key_id.to_string(),
|
||||
pending_window_in_days: Some(7),
|
||||
force_immediate: None,
|
||||
confirm_key_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("deletion should be scheduled");
|
||||
|
||||
+28
-253
@@ -17,18 +17,17 @@
|
||||
use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
|
||||
use crate::backends::KmsBackend;
|
||||
use crate::cache::{KmsCache, KmsCacheStats};
|
||||
use crate::config::{ENV_KMS_ALLOW_IMMEDIATE_DELETION, KmsConfig};
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::config::KmsConfig;
|
||||
use crate::error::Result;
|
||||
use crate::types::{
|
||||
CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse,
|
||||
DEFAULT_PENDING_DELETION_WINDOW_DAYS, DecryptRequest, DecryptResponse, DeleteKeyRequest, DeleteKeyResponse,
|
||||
DescribeKeyRequest, DescribeKeyResponse, EncryptRequest, EncryptResponse, GenerateDataKeyRequest, GenerateDataKeyResponse,
|
||||
ListKeysRequest, ListKeysResponse, MAX_PENDING_DELETION_WINDOW_DAYS, MIN_PENDING_DELETION_WINDOW_DAYS, OperationContext,
|
||||
CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse, DecryptRequest, DecryptResponse,
|
||||
DeleteKeyRequest, DeleteKeyResponse, DescribeKeyRequest, DescribeKeyResponse, EncryptRequest, EncryptResponse,
|
||||
GenerateDataKeyRequest, GenerateDataKeyResponse, ListKeysRequest, ListKeysResponse, OperationContext,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::warn;
|
||||
|
||||
/// KMS Manager coordinates operations between backends and caching
|
||||
#[derive(Clone)]
|
||||
@@ -39,18 +38,12 @@ pub struct KmsManager {
|
||||
enable_cache: bool,
|
||||
backend_kind: &'static str,
|
||||
audit_sink: Option<Arc<dyn KmsAuditSink>>,
|
||||
allow_immediate_deletion: bool,
|
||||
}
|
||||
|
||||
impl KmsManager {
|
||||
/// Create a new KMS manager with the given backend and config
|
||||
pub fn new(backend: Arc<dyn KmsBackend>, config: KmsConfig) -> Self {
|
||||
let cache = Arc::new(RwLock::new(KmsCache::new(config.cache_config.max_keys as u64)));
|
||||
if config.allow_immediate_deletion {
|
||||
warn!(
|
||||
"KMS immediate key deletion is enabled: a DeleteKey request may destroy key material without any waiting window, and every object encrypted under that key becomes permanently unreadable"
|
||||
);
|
||||
}
|
||||
Self {
|
||||
backend,
|
||||
cache,
|
||||
@@ -58,7 +51,6 @@ impl KmsManager {
|
||||
enable_cache: config.enable_cache,
|
||||
backend_kind: config.backend.as_str(),
|
||||
audit_sink: None,
|
||||
allow_immediate_deletion: config.allow_immediate_deletion,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,8 +225,7 @@ impl KmsManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delete a key, either scheduled behind the waiting window or — when the
|
||||
/// server allows it — immediately.
|
||||
/// Delete a key
|
||||
///
|
||||
/// Audited as an internal operation; callers serving an authenticated
|
||||
/// request should use [`Self::delete_key_with_context`].
|
||||
@@ -255,13 +246,7 @@ impl KmsManager {
|
||||
result
|
||||
}
|
||||
|
||||
/// This is the single enforcement point for the waiting window: every
|
||||
/// admin-facing deletion goes through here, so the checks below run before
|
||||
/// any backend sees the request. The backends repeat the window bound as a
|
||||
/// defensive assertion for callers that hold a backend handle directly.
|
||||
async fn delete_key_inner(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
|
||||
self.check_deletion_request(&request)?;
|
||||
|
||||
let response = self.backend.delete_key(request).await?;
|
||||
|
||||
// Remove from cache if enabled and key is being deleted
|
||||
@@ -273,44 +258,6 @@ impl KmsManager {
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// Gate a deletion request before it reaches the backend.
|
||||
///
|
||||
/// Immediate deletion is unrecoverable, so it needs both a server-side
|
||||
/// opt-in and a per-request confirmation that echoes the key id; without
|
||||
/// either, the request is refused rather than downgraded to a scheduled
|
||||
/// deletion, so a caller never believes a key is gone when it is not.
|
||||
fn check_deletion_request(&self, request: &DeleteKeyRequest) -> Result<()> {
|
||||
if !request.force_immediate.unwrap_or(false) {
|
||||
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
|
||||
if !(MIN_PENDING_DELETION_WINDOW_DAYS..=MAX_PENDING_DELETION_WINDOW_DAYS).contains(&days) {
|
||||
return Err(KmsError::invalid_parameter(format!(
|
||||
"pending_window_in_days must be between {MIN_PENDING_DELETION_WINDOW_DAYS} and {MAX_PENDING_DELETION_WINDOW_DAYS}"
|
||||
)));
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if !self.allow_immediate_deletion {
|
||||
return Err(KmsError::invalid_operation(format!(
|
||||
"immediate deletion of key {} is not allowed; schedule the deletion and wait out the pending window, or set {ENV_KMS_ALLOW_IMMEDIATE_DELETION}=true on the server",
|
||||
request.key_id
|
||||
)));
|
||||
}
|
||||
|
||||
if request.confirm_key_id.as_deref() != Some(request.key_id.as_str()) {
|
||||
return Err(KmsError::invalid_operation(format!(
|
||||
"immediate deletion of key {} requires confirm_key_id to repeat the key id exactly",
|
||||
request.key_id
|
||||
)));
|
||||
}
|
||||
|
||||
warn!(
|
||||
key_id = %request.key_id,
|
||||
"immediate KMS key deletion accepted; key material is destroyed without a waiting window and cannot be recovered"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Cancel key deletion
|
||||
///
|
||||
/// Audited as an internal operation; callers serving an authenticated
|
||||
@@ -402,6 +349,27 @@ impl KmsManager {
|
||||
result
|
||||
}
|
||||
|
||||
/// Replace a key's description; `None` clears it
|
||||
pub async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
|
||||
self.backend.update_key_description(key_id, description).await?;
|
||||
self.invalidate_cached_metadata(key_id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Add or overwrite key tags, leaving every other tag untouched
|
||||
pub async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
|
||||
self.backend.tag_key(key_id, tags).await?;
|
||||
self.invalidate_cached_metadata(key_id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove key tags; tags that are not set are ignored
|
||||
pub async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
|
||||
self.backend.untag_key(key_id, tag_keys).await?;
|
||||
self.invalidate_cached_metadata(key_id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drop cached metadata after a state mutation so the next describe
|
||||
/// observes backend truth instead of the pre-mutation snapshot.
|
||||
async fn invalidate_cached_metadata(&self, key_id: &str) {
|
||||
@@ -637,7 +605,6 @@ mod tests {
|
||||
key_id: AUDITED_KEY_ID.to_string(),
|
||||
pending_window_in_days: None,
|
||||
force_immediate: None,
|
||||
confirm_key_id: None,
|
||||
},
|
||||
context,
|
||||
)
|
||||
@@ -1052,196 +1019,4 @@ mod tests {
|
||||
.await
|
||||
.expect("second data key should decrypt with its own context");
|
||||
}
|
||||
|
||||
/// Manager over a local backend, with the immediate-deletion gate set as
|
||||
/// the server operator would set it.
|
||||
async fn deletion_manager(temp_dir: &tempfile::TempDir, allow_immediate_deletion: bool) -> KmsManager {
|
||||
let mut config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
|
||||
config.allow_immediate_deletion = allow_immediate_deletion;
|
||||
let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
|
||||
KmsManager::new(backend, config)
|
||||
}
|
||||
|
||||
async fn create_named_key(manager: &KmsManager, key_name: &str) -> String {
|
||||
manager
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some(key_name.to_string()),
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("Failed to create key")
|
||||
.key_id
|
||||
}
|
||||
|
||||
/// Data key generated up front, decrypted again afterwards: a refused
|
||||
/// deletion must leave the master key material byte-for-byte usable, not
|
||||
/// merely leave a metadata record behind.
|
||||
async fn data_key_probe(manager: &KmsManager, key_id: &str) -> (Vec<u8>, Vec<u8>) {
|
||||
let generated = manager
|
||||
.generate_data_key(GenerateDataKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
key_spec: KeySpec::Aes256,
|
||||
encryption_context: HashMap::new(),
|
||||
})
|
||||
.await
|
||||
.expect("Failed to generate data key");
|
||||
(generated.plaintext_key, generated.ciphertext_blob)
|
||||
}
|
||||
|
||||
async fn assert_key_material_intact(manager: &KmsManager, key_id: &str, probe: &(Vec<u8>, Vec<u8>)) {
|
||||
let state = manager
|
||||
.describe_key(DescribeKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("a key that was not deleted must still be describable")
|
||||
.key_metadata
|
||||
.key_state;
|
||||
assert_eq!(state, KeyState::Enabled, "a refused deletion must not change the key state");
|
||||
|
||||
let decrypted = manager
|
||||
.decrypt(DecryptRequest {
|
||||
ciphertext: probe.1.clone(),
|
||||
encryption_context: HashMap::new(),
|
||||
grant_tokens: Vec::new(),
|
||||
})
|
||||
.await
|
||||
.expect("key material must still decrypt data keys issued before the refused deletion");
|
||||
assert_eq!(decrypted.plaintext, probe.0, "decrypted data key must match the original plaintext");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn immediate_deletion_is_refused_under_default_config() {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let manager = deletion_manager(&temp_dir, false).await;
|
||||
let key_id = create_named_key(&manager, "default-config-force-delete").await;
|
||||
let probe = data_key_probe(&manager, &key_id).await;
|
||||
|
||||
// Confirmation present and correct: the server-side gate alone must
|
||||
// refuse this, no matter how well-formed the request is.
|
||||
let error = manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: Some(key_id.clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("immediate deletion must be refused unless the server allows it");
|
||||
assert!(
|
||||
matches!(error, KmsError::InvalidOperation { .. }),
|
||||
"expected InvalidOperation, got {error:?}"
|
||||
);
|
||||
|
||||
assert_key_material_intact(&manager, &key_id, &probe).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn immediate_deletion_requires_a_matching_confirmation() {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let manager = deletion_manager(&temp_dir, true).await;
|
||||
let key_id = create_named_key(&manager, "confirmation-required").await;
|
||||
let probe = data_key_probe(&manager, &key_id).await;
|
||||
|
||||
for confirmation in [None, Some(String::new()), Some(format!("{key_id}-typo"))] {
|
||||
let result = manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: confirmation.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(KmsError::InvalidOperation { .. })),
|
||||
"confirmation {confirmation:?} must be refused, got {result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
assert_key_material_intact(&manager, &key_id, &probe).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn immediate_deletion_succeeds_with_a_matching_confirmation() {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let manager = deletion_manager(&temp_dir, true).await;
|
||||
let key_id = create_named_key(&manager, "confirmed-force-delete").await;
|
||||
|
||||
manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: Some(key_id.clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("a confirmed immediate deletion must be allowed once the server enables it");
|
||||
|
||||
let error = manager
|
||||
.describe_key(DescribeKeyRequest { key_id: key_id.clone() })
|
||||
.await
|
||||
.expect_err("an immediately deleted key must be gone");
|
||||
assert!(matches!(error, KmsError::KeyNotFound { .. }), "expected KeyNotFound, got {error:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pending_window_outside_the_supported_range_is_refused() {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let manager = deletion_manager(&temp_dir, false).await;
|
||||
let key_id = create_named_key(&manager, "window-bounds").await;
|
||||
let probe = data_key_probe(&manager, &key_id).await;
|
||||
|
||||
for days in [0, MIN_PENDING_DELETION_WINDOW_DAYS - 1, MAX_PENDING_DELETION_WINDOW_DAYS + 1] {
|
||||
let result = manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
pending_window_in_days: Some(days),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(KmsError::InvalidOperation { .. })),
|
||||
"a {days}-day window must be refused, got {result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
assert_key_material_intact(&manager, &key_id, &probe).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduled_deletion_keeps_its_existing_behaviour() {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let manager = deletion_manager(&temp_dir, false).await;
|
||||
|
||||
for (name, days) in [
|
||||
("schedule-default-window", None),
|
||||
("schedule-min-window", Some(MIN_PENDING_DELETION_WINDOW_DAYS)),
|
||||
("schedule-max-window", Some(MAX_PENDING_DELETION_WINDOW_DAYS)),
|
||||
] {
|
||||
let key_id = create_named_key(&manager, name).await;
|
||||
let response = manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
pending_window_in_days: days,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("scheduling a deletion inside the window must still succeed");
|
||||
assert!(response.deletion_date.is_some(), "a scheduled deletion must report its deadline");
|
||||
assert_eq!(response.key_metadata.key_state, KeyState::PendingDeletion);
|
||||
|
||||
manager
|
||||
.cancel_key_deletion(CancelKeyDeletionRequest { key_id: key_id.clone() })
|
||||
.await
|
||||
.expect("a scheduled deletion must still be cancellable");
|
||||
let state = manager
|
||||
.describe_key(DescribeKeyRequest { key_id })
|
||||
.await
|
||||
.expect("describe should succeed")
|
||||
.key_metadata
|
||||
.key_state;
|
||||
assert_eq!(state, KeyState::Enabled, "cancelling must restore the key");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,7 +104,9 @@ pub(crate) fn classify_vaultrs(error: &vaultrs::error::ClientError) -> ErrorClas
|
||||
}
|
||||
}
|
||||
|
||||
fn classify_status(code: u16) -> ErrorClass {
|
||||
/// Retry classification of an HTTP status, shared by every backend that talks
|
||||
/// to an external KMS over HTTP.
|
||||
pub(crate) fn classify_status(code: u16) -> ErrorClass {
|
||||
match code {
|
||||
429 | 500 | 502 | 503 | 504 => ErrorClass::RetryableStatus,
|
||||
_ => ErrorClass::Fatal,
|
||||
|
||||
@@ -14,6 +14,9 @@
|
||||
|
||||
//! Object encryption service for S3-compatible encryption
|
||||
|
||||
use crate::api_types::{
|
||||
TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse, UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
|
||||
};
|
||||
use crate::cache::KmsCacheStats;
|
||||
use crate::encryption::ciphers::{create_cipher, generate_iv};
|
||||
use crate::error::{KmsError, Result};
|
||||
@@ -184,6 +187,64 @@ impl ObjectEncryptionService {
|
||||
self.kms_manager.list_keys_with_context(request, context).await
|
||||
}
|
||||
|
||||
/// Replace a master key's description (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - UpdateKeyDescriptionRequest with key ID and the new
|
||||
/// description; an empty description clears the stored value
|
||||
///
|
||||
/// # Returns
|
||||
/// UpdateKeyDescriptionResponse acknowledging the update
|
||||
///
|
||||
pub async fn update_key_description(&self, request: UpdateKeyDescriptionRequest) -> Result<UpdateKeyDescriptionResponse> {
|
||||
let description = (!request.description.is_empty()).then_some(request.description.as_str());
|
||||
self.kms_manager.update_key_description(&request.key_id, description).await?;
|
||||
|
||||
Ok(UpdateKeyDescriptionResponse {
|
||||
success: true,
|
||||
message: "key description updated".to_string(),
|
||||
key_id: request.key_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Add or overwrite master key tags (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - TagKeyRequest with key ID and the tags to set; tags
|
||||
/// outside the request are left untouched
|
||||
///
|
||||
/// # Returns
|
||||
/// TagKeyResponse acknowledging the update
|
||||
///
|
||||
pub async fn tag_key(&self, request: TagKeyRequest) -> Result<TagKeyResponse> {
|
||||
self.kms_manager.tag_key(&request.key_id, &request.tags).await?;
|
||||
|
||||
Ok(TagKeyResponse {
|
||||
success: true,
|
||||
message: "key tags updated".to_string(),
|
||||
key_id: request.key_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Remove master key tags (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - UntagKeyRequest with key ID and the tag keys to remove;
|
||||
/// tag keys that are not set are ignored, so the call is idempotent
|
||||
///
|
||||
/// # Returns
|
||||
/// UntagKeyResponse acknowledging the removal
|
||||
///
|
||||
pub async fn untag_key(&self, request: UntagKeyRequest) -> Result<UntagKeyResponse> {
|
||||
self.kms_manager.untag_key(&request.key_id, &request.tag_keys).await?;
|
||||
|
||||
Ok(UntagKeyResponse {
|
||||
success: true,
|
||||
message: "key tags removed".to_string(),
|
||||
key_id: request.key_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Generate a data encryption key (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -966,6 +1027,128 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn describe(service: &ObjectEncryptionService, key_id: &str) -> KeyMetadata {
|
||||
service
|
||||
.describe_key(DescribeKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("describe should succeed")
|
||||
.key_metadata
|
||||
}
|
||||
|
||||
async fn create_metadata_test_key(service: &ObjectEncryptionService, key_id: &str) {
|
||||
service
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some(key_id.to_string()),
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: Some("original".to_string()),
|
||||
policy: None,
|
||||
tags: HashMap::from([
|
||||
("name".to_string(), key_id.to_string()),
|
||||
("team".to_string(), "storage".to_string()),
|
||||
]),
|
||||
origin: None,
|
||||
})
|
||||
.await
|
||||
.expect("test key should be created");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn key_metadata_updates_are_visible_to_describe() {
|
||||
let (service, _temp_dir) = create_test_service().await;
|
||||
create_metadata_test_key(&service, "metadata-key").await;
|
||||
|
||||
service
|
||||
.update_key_description(UpdateKeyDescriptionRequest {
|
||||
key_id: "metadata-key".to_string(),
|
||||
description: "updated".to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("description update should succeed");
|
||||
service
|
||||
.tag_key(TagKeyRequest {
|
||||
key_id: "metadata-key".to_string(),
|
||||
tags: HashMap::from([
|
||||
("team".to_string(), "platform".to_string()),
|
||||
("env".to_string(), "prod".to_string()),
|
||||
]),
|
||||
})
|
||||
.await
|
||||
.expect("tagging should succeed");
|
||||
|
||||
// Reading through the manager's metadata cache must observe the
|
||||
// updates, not the snapshot cached when the key was created.
|
||||
let metadata = describe(&service, "metadata-key").await;
|
||||
assert_eq!(metadata.description.as_deref(), Some("updated"));
|
||||
assert_eq!(metadata.tags.get("team").map(String::as_str), Some("platform"));
|
||||
assert_eq!(metadata.tags.get("env").map(String::as_str), Some("prod"));
|
||||
assert_eq!(
|
||||
metadata.tags.get("name").map(String::as_str),
|
||||
Some("metadata-key"),
|
||||
"tags set at creation must survive a later tag update"
|
||||
);
|
||||
|
||||
// Untagging is idempotent: removing an absent tag is a no-op, not an
|
||||
// error, so a retried request stays safe.
|
||||
for attempt in 1..=2 {
|
||||
service
|
||||
.untag_key(UntagKeyRequest {
|
||||
key_id: "metadata-key".to_string(),
|
||||
tag_keys: vec!["env".to_string()],
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("untag attempt {attempt} should succeed: {error:?}"));
|
||||
}
|
||||
let metadata = describe(&service, "metadata-key").await;
|
||||
assert!(!metadata.tags.contains_key("env"), "untagged tag must be gone: {:?}", metadata.tags);
|
||||
assert_eq!(
|
||||
metadata.tags.get("team").map(String::as_str),
|
||||
Some("platform"),
|
||||
"untagging must not touch other tags"
|
||||
);
|
||||
|
||||
// An empty description clears the stored value.
|
||||
service
|
||||
.update_key_description(UpdateKeyDescriptionRequest {
|
||||
key_id: "metadata-key".to_string(),
|
||||
description: String::new(),
|
||||
})
|
||||
.await
|
||||
.expect("clearing the description should succeed");
|
||||
assert_eq!(describe(&service, "metadata-key").await.description, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn identity_tag_is_not_writable_through_metadata_updates() {
|
||||
let (service, _temp_dir) = create_test_service().await;
|
||||
create_metadata_test_key(&service, "identity-key").await;
|
||||
|
||||
let rewrite = service
|
||||
.tag_key(TagKeyRequest {
|
||||
key_id: "identity-key".to_string(),
|
||||
tags: HashMap::from([("name".to_string(), "other-key".to_string())]),
|
||||
})
|
||||
.await
|
||||
.expect_err("rewriting the identity tag must be rejected");
|
||||
assert!(matches!(rewrite, KmsError::InvalidOperation { .. }), "got {rewrite:?}");
|
||||
|
||||
let removal = service
|
||||
.untag_key(UntagKeyRequest {
|
||||
key_id: "identity-key".to_string(),
|
||||
tag_keys: vec!["team".to_string(), "name".to_string()],
|
||||
})
|
||||
.await
|
||||
.expect_err("removing the identity tag must be rejected");
|
||||
assert!(matches!(removal, KmsError::InvalidOperation { .. }), "got {removal:?}");
|
||||
|
||||
// Both rejections are pre-write: the record is untouched, including
|
||||
// the ordinary tag that shared the rejected untag request.
|
||||
let metadata = describe(&service, "identity-key").await;
|
||||
assert_eq!(metadata.tags.get("name").map(String::as_str), Some("identity-key"));
|
||||
assert_eq!(metadata.tags.get("team").map(String::as_str), Some("storage"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_decrypt_data_key_uses_object_encryption_context() {
|
||||
let (service, _temp_dir) = create_test_service().await;
|
||||
|
||||
@@ -626,6 +626,11 @@ impl KmsServiceManager {
|
||||
let backend = crate::backends::static_kms::StaticKmsBackend::new(config.clone()).await?;
|
||||
Arc::new(backend) as Arc<dyn KmsBackend>
|
||||
}
|
||||
BackendConfig::Aws(_) => {
|
||||
info!("Creating AWS KMS backend for version {}", version);
|
||||
let backend = crate::backends::aws::AwsKmsBackend::new(config.clone()).await?;
|
||||
Arc::new(backend) as Arc<dyn KmsBackend>
|
||||
}
|
||||
};
|
||||
|
||||
// Create KMS manager
|
||||
|
||||
+3
-25
@@ -893,37 +893,15 @@ impl std::str::FromStr for EncryptionAlgorithm {
|
||||
}
|
||||
}
|
||||
|
||||
/// Shortest pending-deletion waiting window a caller may ask for, in days.
|
||||
///
|
||||
/// The window is enforced once, in [`crate::manager::KmsManager::delete_key`];
|
||||
/// backends keep the same bound as a defensive check for direct callers.
|
||||
pub const MIN_PENDING_DELETION_WINDOW_DAYS: u32 = 7;
|
||||
|
||||
/// Longest pending-deletion waiting window a caller may ask for, in days.
|
||||
pub const MAX_PENDING_DELETION_WINDOW_DAYS: u32 = 30;
|
||||
|
||||
/// Waiting window applied when a delete request does not name one.
|
||||
pub const DEFAULT_PENDING_DELETION_WINDOW_DAYS: u32 = MAX_PENDING_DELETION_WINDOW_DAYS;
|
||||
|
||||
/// Request to delete a key
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DeleteKeyRequest {
|
||||
/// Key ID to delete
|
||||
pub key_id: String,
|
||||
/// Number of days to wait before deletion (7-30 days, optional, defaults to 30)
|
||||
/// Number of days to wait before deletion (7-30 days, optional)
|
||||
pub pending_window_in_days: Option<u32>,
|
||||
/// Destroy the key material right away instead of scheduling it.
|
||||
///
|
||||
/// Refused unless the server enables `allow_immediate_deletion`; see
|
||||
/// [`crate::manager::KmsManager::delete_key`] for the gate.
|
||||
/// Force immediate deletion (for development/testing only)
|
||||
pub force_immediate: Option<bool>,
|
||||
/// Key id echoed back by the caller to confirm an immediate deletion.
|
||||
///
|
||||
/// Must equal `key_id` exactly whenever `force_immediate` is set. Optional
|
||||
/// with a serde default because this type is part of the admin API
|
||||
/// contract: clients that never ask for immediate deletion are unaffected.
|
||||
#[serde(default)]
|
||||
pub confirm_key_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Response from delete key operation
|
||||
|
||||
@@ -146,6 +146,7 @@ tracing-opentelemetry = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["fmt", "env-filter", "tracing-log", "local-time", "json"] }
|
||||
tokio = { workspace = true, features = ["sync", "rt-multi-thread", "time", "macros"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
url.workspace = true
|
||||
dial9-tokio-telemetry = { workspace = true, optional = true }
|
||||
thiserror = { workspace = true }
|
||||
zstd = { workspace = true, features = ["zstdmt"] }
|
||||
@@ -162,3 +163,4 @@ libc = { workspace = true }
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
log = "0.4"
|
||||
|
||||
@@ -16,22 +16,25 @@
|
||||
|
||||
use crate::metrics::report::PrometheusMetric;
|
||||
use crate::metrics::schema::bucket_replication::{
|
||||
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD,
|
||||
BUCKET_REPL_LAST_HR_FAILED_COUNT_MD, BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD,
|
||||
BUCKET_REPL_LATENCY_MS_MD, BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD,
|
||||
BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD,
|
||||
BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD,
|
||||
BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD,
|
||||
BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD,
|
||||
BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD,
|
||||
BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD,
|
||||
BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, BUCKET_REPL_RESYNC_FAILED_TOTAL_MD,
|
||||
BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD,
|
||||
BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, TARGET_ARN_L,
|
||||
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD,
|
||||
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
|
||||
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, BUCKET_REPL_LAST_HR_FAILED_COUNT_MD,
|
||||
BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD, BUCKET_REPL_LATENCY_MS_MD,
|
||||
BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD,
|
||||
BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD,
|
||||
BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L,
|
||||
TARGET_ARN_L,
|
||||
};
|
||||
use std::borrow::Cow;
|
||||
|
||||
const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 25;
|
||||
const BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET: usize = 5;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct BucketReplicationTargetStats {
|
||||
@@ -80,6 +83,16 @@ pub struct BucketReplicationStats {
|
||||
pub targets: Vec<BucketReplicationTargetStats>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct BucketReplicationBacklogStats {
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) current_backlog_count: u64,
|
||||
pub(crate) current_backlog_bytes: u64,
|
||||
pub(crate) durable_mrf_available: bool,
|
||||
pub(crate) durable_mrf_backlog_count: u64,
|
||||
pub(crate) durable_mrf_backlog_bytes: u64,
|
||||
}
|
||||
|
||||
pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
|
||||
if stats.is_empty() {
|
||||
return Vec::new();
|
||||
@@ -249,7 +262,6 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, stat.resync_duration_ms as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone()),
|
||||
);
|
||||
|
||||
for target in &stat.targets {
|
||||
let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
|
||||
metrics.push(
|
||||
@@ -265,6 +277,43 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
|
||||
metrics
|
||||
}
|
||||
|
||||
pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicationBacklogStats]) -> Vec<PrometheusMetric> {
|
||||
if stats.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut metrics = Vec::with_capacity(stats.len() * BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET);
|
||||
for stat in stats {
|
||||
let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
|
||||
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, stat.current_backlog_count as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD, stat.current_backlog_bytes as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(
|
||||
&BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD,
|
||||
if stat.durable_mrf_available { 1.0 } else { 0.0 },
|
||||
)
|
||||
.with_label(BUCKET_L, bucket_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, stat.durable_mrf_backlog_count as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD, stat.durable_mrf_backlog_bytes as f64)
|
||||
.with_label(BUCKET_L, bucket_label),
|
||||
);
|
||||
}
|
||||
|
||||
metrics
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -369,6 +418,56 @@ mod tests {
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_bucket_replication_backlog_metrics() {
|
||||
let stats = vec![BucketReplicationBacklogStats {
|
||||
bucket: "b1".to_string(),
|
||||
current_backlog_count: 3,
|
||||
current_backlog_bytes: 4096,
|
||||
durable_mrf_available: true,
|
||||
durable_mrf_backlog_count: 2,
|
||||
durable_mrf_backlog_bytes: 2048,
|
||||
}];
|
||||
|
||||
let metrics = collect_bucket_replication_backlog_metrics(&stats);
|
||||
assert_eq!(metrics.len(), 5);
|
||||
|
||||
let backlog_count_name = BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == backlog_count_name
|
||||
&& metric.value == 3.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
}));
|
||||
|
||||
let backlog_bytes_name = BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == backlog_bytes_name
|
||||
&& metric.value == 4096.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
}));
|
||||
|
||||
let durable_available_name = BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == durable_available_name
|
||||
&& metric.value == 1.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
}));
|
||||
|
||||
let durable_count_name = BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == durable_count_name
|
||||
&& metric.value == 2.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
}));
|
||||
|
||||
let durable_bytes_name = BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == durable_bytes_name
|
||||
&& metric.value == 2048.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_bucket_replication_metrics_empty() {
|
||||
let stats: Vec<BucketReplicationStats> = Vec::new();
|
||||
@@ -376,6 +475,41 @@ mod tests {
|
||||
assert!(metrics.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backlog_metrics_are_bucket_scoped_without_target_labels() {
|
||||
let stats = vec![BucketReplicationBacklogStats {
|
||||
bucket: "scope-bucket".to_string(),
|
||||
current_backlog_count: 5,
|
||||
current_backlog_bytes: 8192,
|
||||
durable_mrf_available: true,
|
||||
durable_mrf_backlog_count: 2,
|
||||
durable_mrf_backlog_bytes: 4096,
|
||||
}];
|
||||
|
||||
let metrics = collect_bucket_replication_backlog_metrics(&stats);
|
||||
let backlog_names = [
|
||||
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD.get_full_metric_name(),
|
||||
BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD.get_full_metric_name(),
|
||||
BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD.get_full_metric_name(),
|
||||
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD.get_full_metric_name(),
|
||||
BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD.get_full_metric_name(),
|
||||
];
|
||||
|
||||
for name in backlog_names {
|
||||
let metric = metrics
|
||||
.iter()
|
||||
.find(|metric| metric.name == name)
|
||||
.expect("backlog metric should be emitted");
|
||||
assert!(
|
||||
metric
|
||||
.labels
|
||||
.iter()
|
||||
.any(|(key, value)| *key == BUCKET_L && value == "scope-bucket")
|
||||
);
|
||||
assert!(!metric.labels.iter().any(|(key, _)| *key == TARGET_ARN_L));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_bucket_replication_bandwidth_metrics() {
|
||||
let stats = vec![BucketReplicationBandwidthStats {
|
||||
|
||||
@@ -42,6 +42,7 @@ pub mod system_process;
|
||||
|
||||
pub use audit::{AuditTargetStats, collect_audit_metrics};
|
||||
pub use bucket::{BucketStats, collect_bucket_metrics};
|
||||
pub(crate) use bucket_replication::{BucketReplicationBacklogStats, collect_bucket_replication_backlog_metrics};
|
||||
pub use bucket_replication::{
|
||||
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
|
||||
collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
|
||||
|
||||
@@ -31,9 +31,9 @@ pub use scheduler::{
|
||||
metrics_runtime_controller_snapshot, metrics_runtime_status_snapshot,
|
||||
};
|
||||
pub(crate) use storage_api::metrics::{
|
||||
BucketOperations, BucketOptions, ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsStore, StorageAdminApi,
|
||||
obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config,
|
||||
obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
|
||||
BucketOperations, BucketOptions, ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore,
|
||||
StorageAdminApi, obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor,
|
||||
obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
|
||||
obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
|
||||
obs_resolve_object_store_handle, obs_transition_state_handle,
|
||||
};
|
||||
|
||||
@@ -35,6 +35,7 @@ use crate::metrics::collectors::{
|
||||
ProcessMemoryStats,
|
||||
collect_audit_metrics,
|
||||
collect_bucket_metrics,
|
||||
collect_bucket_replication_backlog_metrics,
|
||||
collect_bucket_replication_bandwidth_metrics,
|
||||
collect_bucket_replication_metrics,
|
||||
collect_bucket_usage_metrics,
|
||||
@@ -94,7 +95,7 @@ use crate::metrics::schema::notification_target::{
|
||||
};
|
||||
use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL};
|
||||
use crate::metrics::stats_collector::{
|
||||
ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_detail_stats,
|
||||
ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_stats_bundle,
|
||||
collect_bucket_stats, collect_cluster_and_health_stats, collect_cluster_config_stats, collect_cluster_usage_metric_stats,
|
||||
collect_compression_cluster_stats, collect_disk_and_system_drive_stats, collect_erasure_set_stats,
|
||||
collect_host_network_stats, collect_iam_stats, collect_ilm_metric_stats, collect_internode_network_stats,
|
||||
@@ -1348,8 +1349,9 @@ pub fn init_metrics_runtime(token: CancellationToken) {
|
||||
// Phase-1 action: force zero for removed keys during tombstone cycles.
|
||||
metrics.extend(collect_repl_bw_zero_tombstone_metrics(&zero_tombstones));
|
||||
|
||||
let bucket_replication = collect_bucket_replication_detail_stats().await;
|
||||
let (bucket_replication, bucket_replication_backlog) = collect_bucket_replication_stats_bundle().await;
|
||||
metrics.extend(collect_bucket_replication_metrics(&bucket_replication));
|
||||
metrics.extend(collect_bucket_replication_backlog_metrics(&bucket_replication_backlog));
|
||||
let replication = collect_replication_stats().await;
|
||||
metrics.extend(collect_replication_metrics(&replication));
|
||||
report_metrics(&metrics);
|
||||
|
||||
@@ -33,6 +33,11 @@ const RESYNC_COMPLETED_TOTAL: &str = "resync_completed_total";
|
||||
const RESYNC_FAILED_TOTAL: &str = "resync_failed_total";
|
||||
const RESYNC_CANCELED_TOTAL: &str = "resync_canceled_total";
|
||||
const RESYNC_DURATION_MS_TOTAL: &str = "resync_duration_ms_total";
|
||||
const CURRENT_BACKLOG_COUNT: &str = "current_backlog_count";
|
||||
const CURRENT_BACKLOG_BYTES: &str = "current_backlog_bytes";
|
||||
const DURABLE_MRF_AVAILABLE: &str = "durable_mrf_available";
|
||||
const DURABLE_MRF_BACKLOG_COUNT: &str = "durable_mrf_backlog_count";
|
||||
const DURABLE_MRF_BACKLOG_BYTES: &str = "durable_mrf_backlog_bytes";
|
||||
|
||||
pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
@@ -79,6 +84,51 @@ pub static BUCKET_REPL_LATENCY_MS_MD: LazyLock<MetricDescriptor> = LazyLock::new
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(CURRENT_BACKLOG_BYTES),
|
||||
"Current number of bytes admitted to the in-memory replication worker queues for a bucket",
|
||||
&[BUCKET_L],
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(CURRENT_BACKLOG_COUNT),
|
||||
"Current number of objects admitted to the in-memory replication worker queues for a bucket",
|
||||
&[BUCKET_L],
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(DURABLE_MRF_AVAILABLE),
|
||||
"Whether the durable MRF backlog snapshot is available for this bucket on this node",
|
||||
&[BUCKET_L],
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(DURABLE_MRF_BACKLOG_COUNT),
|
||||
"Current number of objects in the durable MRF backlog file for a bucket on this node",
|
||||
&[BUCKET_L],
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(DURABLE_MRF_BACKLOG_BYTES),
|
||||
"Current bytes in the durable MRF backlog file for a bucket on this node",
|
||||
&[BUCKET_L],
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::ProxiedDeleteTaggingRequestsTotal,
|
||||
|
||||
@@ -128,7 +128,7 @@ pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = L
|
||||
pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationRecentBacklogCount,
|
||||
"Total number of objects currently in replication backlog (failed plus queued)",
|
||||
"Legacy replication backlog indicator: failed target objects plus objects currently queued on this node",
|
||||
&[],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
|
||||
@@ -21,17 +21,18 @@
|
||||
//! and convert them to the Stats structs used by collectors.
|
||||
|
||||
use crate::metrics::collectors::{
|
||||
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats, BucketStats, BucketUsageStats,
|
||||
ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats,
|
||||
DriveCountStats, DriveDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats,
|
||||
ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats, ScannerStats,
|
||||
BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
|
||||
BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats,
|
||||
CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats, HostNetworkStats,
|
||||
IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats,
|
||||
ScannerStats,
|
||||
};
|
||||
use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
|
||||
use crate::metrics::{
|
||||
BucketOperations, BucketOptions, ObsEcstoreResult, ObsStore, StorageAdminApi, obs_bucket_replication_stats_snapshot,
|
||||
obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free,
|
||||
obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
|
||||
obs_resolve_object_store_handle,
|
||||
BucketOperations, BucketOptions, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore, StorageAdminApi,
|
||||
obs_bucket_replication_stats_snapshot, obs_get_quota_config, obs_get_total_usable_capacity,
|
||||
obs_get_total_usable_capacity_free, obs_load_compression_total_from_memory, obs_load_data_usage_from_backend,
|
||||
obs_replication_site_stats_snapshot, obs_resolve_object_store_handle,
|
||||
};
|
||||
use crate::node_identity::current_local_node_identity;
|
||||
use chrono::Utc;
|
||||
@@ -192,49 +193,65 @@ async fn obs_ilm_runtime_snapshot() -> ObsIlmRuntimeSnapshot {
|
||||
ilm_runtime_snapshot().await
|
||||
}
|
||||
|
||||
async fn obs_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
|
||||
obs_bucket_replication_stats_snapshot()
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|stats| BucketReplicationStats {
|
||||
bucket: stats.bucket,
|
||||
total_failed_bytes: stats.total_failed_bytes,
|
||||
total_failed_count: stats.total_failed_count,
|
||||
last_min_failed_bytes: stats.last_min_failed_bytes,
|
||||
last_min_failed_count: stats.last_min_failed_count,
|
||||
last_hour_failed_bytes: stats.last_hour_failed_bytes,
|
||||
last_hour_failed_count: stats.last_hour_failed_count,
|
||||
sent_bytes: stats.sent_bytes,
|
||||
sent_count: stats.sent_count,
|
||||
proxied_get_requests_total: stats.proxied_get_requests_total,
|
||||
proxied_get_requests_failures: stats.proxied_get_requests_failures,
|
||||
proxied_head_requests_total: stats.proxied_head_requests_total,
|
||||
proxied_head_requests_failures: stats.proxied_head_requests_failures,
|
||||
proxied_put_requests_total: stats.proxied_put_requests_total,
|
||||
proxied_put_requests_failures: stats.proxied_put_requests_failures,
|
||||
proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
|
||||
proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
|
||||
proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
|
||||
proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
|
||||
proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
|
||||
proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
|
||||
resync_started_count: stats.resync_started_count,
|
||||
resync_completed_count: stats.resync_completed_count,
|
||||
resync_failed_count: stats.resync_failed_count,
|
||||
resync_canceled_count: stats.resync_canceled_count,
|
||||
resync_duration_ms: stats.resync_duration_ms,
|
||||
targets: stats
|
||||
.targets
|
||||
.into_iter()
|
||||
.map(|target| BucketReplicationTargetStats {
|
||||
target_arn: target.target_arn,
|
||||
bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
|
||||
current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
|
||||
latency_ms: target.latency_ms,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
.collect()
|
||||
async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>) {
|
||||
let snapshots = obs_bucket_replication_stats_snapshot().await;
|
||||
let mut detail_stats = Vec::with_capacity(snapshots.len());
|
||||
let mut backlog_stats = Vec::with_capacity(snapshots.len());
|
||||
|
||||
for stats in snapshots {
|
||||
backlog_stats.push(BucketReplicationBacklogStats {
|
||||
bucket: stats.bucket.clone(),
|
||||
current_backlog_count: stats.current_backlog_count,
|
||||
current_backlog_bytes: stats.current_backlog_bytes,
|
||||
durable_mrf_available: stats.durable_mrf_available,
|
||||
durable_mrf_backlog_count: stats.durable_mrf_backlog_count,
|
||||
durable_mrf_backlog_bytes: stats.durable_mrf_backlog_bytes,
|
||||
});
|
||||
detail_stats.push(bucket_replication_detail_from_snapshot(stats));
|
||||
}
|
||||
|
||||
(detail_stats, backlog_stats)
|
||||
}
|
||||
|
||||
fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnapshot) -> BucketReplicationStats {
|
||||
BucketReplicationStats {
|
||||
bucket: stats.bucket,
|
||||
total_failed_bytes: stats.total_failed_bytes,
|
||||
total_failed_count: stats.total_failed_count,
|
||||
last_min_failed_bytes: stats.last_min_failed_bytes,
|
||||
last_min_failed_count: stats.last_min_failed_count,
|
||||
last_hour_failed_bytes: stats.last_hour_failed_bytes,
|
||||
last_hour_failed_count: stats.last_hour_failed_count,
|
||||
sent_bytes: stats.sent_bytes,
|
||||
sent_count: stats.sent_count,
|
||||
proxied_get_requests_total: stats.proxied_get_requests_total,
|
||||
proxied_get_requests_failures: stats.proxied_get_requests_failures,
|
||||
proxied_head_requests_total: stats.proxied_head_requests_total,
|
||||
proxied_head_requests_failures: stats.proxied_head_requests_failures,
|
||||
proxied_put_requests_total: stats.proxied_put_requests_total,
|
||||
proxied_put_requests_failures: stats.proxied_put_requests_failures,
|
||||
proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
|
||||
proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
|
||||
proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
|
||||
proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
|
||||
proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
|
||||
proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
|
||||
resync_started_count: stats.resync_started_count,
|
||||
resync_completed_count: stats.resync_completed_count,
|
||||
resync_failed_count: stats.resync_failed_count,
|
||||
resync_canceled_count: stats.resync_canceled_count,
|
||||
resync_duration_ms: stats.resync_duration_ms,
|
||||
targets: stats
|
||||
.targets
|
||||
.into_iter()
|
||||
.map(|target| BucketReplicationTargetStats {
|
||||
target_arn: target.target_arn,
|
||||
bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
|
||||
current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
|
||||
latency_ms: target.latency_ms,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn obs_site_replication_stats() -> ReplicationStats {
|
||||
@@ -526,7 +543,16 @@ pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBand
|
||||
|
||||
/// Collect bucket and target level replication stats from the global replication runtime.
|
||||
pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
|
||||
obs_bucket_replication_detail_stats().await
|
||||
obs_bucket_replication_stats_snapshot()
|
||||
.await
|
||||
.into_iter()
|
||||
.map(bucket_replication_detail_from_snapshot)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn collect_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>)
|
||||
{
|
||||
obs_bucket_replication_stats_bundle().await
|
||||
}
|
||||
|
||||
/// Collect site-level replication stats from the global replication runtime.
|
||||
|
||||
@@ -12,11 +12,14 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
|
||||
use rustfs_ecstore::api::bucket::replication::get_global_replication_stats;
|
||||
use rustfs_ecstore::api::bucket::replication::{
|
||||
DurableMrfBucketBacklog, durable_mrf_backlog_summary_snapshot, get_global_replication_stats,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::capacity::{
|
||||
get_total_usable_capacity as obs_get_total_usable_capacity,
|
||||
get_total_usable_capacity_free as obs_get_total_usable_capacity_free,
|
||||
@@ -68,9 +71,50 @@ pub(crate) struct ObsBucketReplicationStatsSnapshot {
|
||||
pub(crate) resync_failed_count: u64,
|
||||
pub(crate) resync_canceled_count: u64,
|
||||
pub(crate) resync_duration_ms: u64,
|
||||
pub(crate) current_backlog_count: u64,
|
||||
pub(crate) current_backlog_bytes: u64,
|
||||
pub(crate) durable_mrf_available: bool,
|
||||
pub(crate) durable_mrf_backlog_count: u64,
|
||||
pub(crate) durable_mrf_backlog_bytes: u64,
|
||||
pub(crate) targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
struct ObsBucketReplicationRuntimeSnapshot {
|
||||
total_failed_bytes: u64,
|
||||
total_failed_count: u64,
|
||||
last_min_failed_bytes: u64,
|
||||
last_min_failed_count: u64,
|
||||
last_hour_failed_bytes: u64,
|
||||
last_hour_failed_count: u64,
|
||||
sent_bytes: u64,
|
||||
sent_count: u64,
|
||||
resync_started_count: u64,
|
||||
resync_completed_count: u64,
|
||||
resync_failed_count: u64,
|
||||
resync_canceled_count: u64,
|
||||
resync_duration_ms: u64,
|
||||
current_backlog_count: u64,
|
||||
current_backlog_bytes: u64,
|
||||
targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
struct ObsBucketReplicationProxySnapshot {
|
||||
proxied_get_requests_total: u64,
|
||||
proxied_get_requests_failures: u64,
|
||||
proxied_head_requests_total: u64,
|
||||
proxied_head_requests_failures: u64,
|
||||
proxied_put_requests_total: u64,
|
||||
proxied_put_requests_failures: u64,
|
||||
proxied_put_tagging_requests_total: u64,
|
||||
proxied_put_tagging_requests_failures: u64,
|
||||
proxied_get_tagging_requests_total: u64,
|
||||
proxied_get_tagging_requests_failures: u64,
|
||||
proxied_delete_tagging_requests_total: u64,
|
||||
proxied_delete_tagging_requests_failures: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub(crate) struct ObsReplicationSiteStatsSnapshot {
|
||||
pub(crate) average_active_workers: f64,
|
||||
@@ -102,59 +146,85 @@ fn replication_backlog_count(failed_counts: impl Iterator<Item = i64>, queued_co
|
||||
failed_backlog.saturating_add(i64_to_u64_floor_zero(queued_count))
|
||||
}
|
||||
|
||||
fn bucket_replication_stats_snapshot_from_parts(
|
||||
bucket: String,
|
||||
runtime: ObsBucketReplicationRuntimeSnapshot,
|
||||
proxy: ObsBucketReplicationProxySnapshot,
|
||||
durable_mrf_available: bool,
|
||||
durable_bucket: DurableMrfBucketBacklog,
|
||||
) -> ObsBucketReplicationStatsSnapshot {
|
||||
ObsBucketReplicationStatsSnapshot {
|
||||
bucket,
|
||||
total_failed_bytes: runtime.total_failed_bytes,
|
||||
total_failed_count: runtime.total_failed_count,
|
||||
last_min_failed_bytes: runtime.last_min_failed_bytes,
|
||||
last_min_failed_count: runtime.last_min_failed_count,
|
||||
last_hour_failed_bytes: runtime.last_hour_failed_bytes,
|
||||
last_hour_failed_count: runtime.last_hour_failed_count,
|
||||
sent_bytes: runtime.sent_bytes,
|
||||
sent_count: runtime.sent_count,
|
||||
proxied_get_requests_total: proxy.proxied_get_requests_total,
|
||||
proxied_get_requests_failures: proxy.proxied_get_requests_failures,
|
||||
proxied_head_requests_total: proxy.proxied_head_requests_total,
|
||||
proxied_head_requests_failures: proxy.proxied_head_requests_failures,
|
||||
proxied_put_requests_total: proxy.proxied_put_requests_total,
|
||||
proxied_put_requests_failures: proxy.proxied_put_requests_failures,
|
||||
proxied_put_tagging_requests_total: proxy.proxied_put_tagging_requests_total,
|
||||
proxied_put_tagging_requests_failures: proxy.proxied_put_tagging_requests_failures,
|
||||
proxied_get_tagging_requests_total: proxy.proxied_get_tagging_requests_total,
|
||||
proxied_get_tagging_requests_failures: proxy.proxied_get_tagging_requests_failures,
|
||||
proxied_delete_tagging_requests_total: proxy.proxied_delete_tagging_requests_total,
|
||||
proxied_delete_tagging_requests_failures: proxy.proxied_delete_tagging_requests_failures,
|
||||
resync_started_count: runtime.resync_started_count,
|
||||
resync_completed_count: runtime.resync_completed_count,
|
||||
resync_failed_count: runtime.resync_failed_count,
|
||||
resync_canceled_count: runtime.resync_canceled_count,
|
||||
resync_duration_ms: runtime.resync_duration_ms,
|
||||
current_backlog_count: runtime.current_backlog_count,
|
||||
current_backlog_bytes: runtime.current_backlog_bytes,
|
||||
durable_mrf_available,
|
||||
durable_mrf_backlog_count: durable_bucket.count,
|
||||
durable_mrf_backlog_bytes: durable_bucket.bytes,
|
||||
targets: runtime.targets,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketReplicationStatsSnapshot> {
|
||||
let Some(stats) = get_global_replication_stats() else {
|
||||
return Vec::new();
|
||||
let stats = get_global_replication_stats();
|
||||
let all_bucket_stats = if let Some(stats) = &stats {
|
||||
stats.get_all().await
|
||||
} else {
|
||||
HashMap::new()
|
||||
};
|
||||
let durable_mrf_summary = if obs_resolve_object_store_handle().is_some() {
|
||||
durable_mrf_backlog_summary_snapshot()
|
||||
} else {
|
||||
Default::default()
|
||||
};
|
||||
let durable_mrf_available = durable_mrf_summary.available;
|
||||
let durable_buckets = durable_mrf_summary
|
||||
.buckets
|
||||
.into_iter()
|
||||
.map(|bucket| (bucket.bucket.clone(), bucket))
|
||||
.collect::<HashMap<String, DurableMrfBucketBacklog>>();
|
||||
let mut bucket_names = Vec::with_capacity(all_bucket_stats.len().saturating_add(durable_buckets.len()));
|
||||
bucket_names.extend(all_bucket_stats.keys().cloned());
|
||||
bucket_names.extend(
|
||||
durable_buckets
|
||||
.keys()
|
||||
.filter(|bucket| !all_bucket_stats.contains_key(*bucket))
|
||||
.cloned(),
|
||||
);
|
||||
let mut buckets = Vec::with_capacity(bucket_names.len());
|
||||
|
||||
let all_bucket_stats = stats.get_all().await;
|
||||
let mut buckets = Vec::with_capacity(all_bucket_stats.len());
|
||||
|
||||
for (bucket, bucket_stats) in all_bucket_stats {
|
||||
let proxy = stats.get_proxy_stats(&bucket).await;
|
||||
let mut total_failed_bytes = 0u64;
|
||||
let mut total_failed_count = 0u64;
|
||||
let mut last_min_failed_bytes = 0u64;
|
||||
let mut last_min_failed_count = 0u64;
|
||||
let mut last_hour_failed_bytes = 0u64;
|
||||
let mut last_hour_failed_count = 0u64;
|
||||
let mut sent_bytes = 0u64;
|
||||
let mut sent_count = 0u64;
|
||||
let mut targets = Vec::with_capacity(bucket_stats.stats.len());
|
||||
|
||||
for (target_arn, target_stats) in bucket_stats.stats {
|
||||
total_failed_bytes = total_failed_bytes.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
|
||||
total_failed_count = total_failed_count.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
|
||||
|
||||
let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
|
||||
last_min_failed_bytes = last_min_failed_bytes.saturating_add(i64_to_u64_floor_zero(last_min.size));
|
||||
last_min_failed_count = last_min_failed_count.saturating_add(i64_to_u64_floor_zero(last_min.count));
|
||||
|
||||
let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
|
||||
last_hour_failed_bytes = last_hour_failed_bytes.saturating_add(i64_to_u64_floor_zero(last_hour.size));
|
||||
last_hour_failed_count = last_hour_failed_count.saturating_add(i64_to_u64_floor_zero(last_hour.count));
|
||||
|
||||
sent_bytes = sent_bytes.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
|
||||
sent_count = sent_count.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
|
||||
|
||||
targets.push(ObsBucketReplicationTargetStatsSnapshot {
|
||||
target_arn,
|
||||
bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
|
||||
current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
|
||||
latency_ms: target_stats.latency.curr,
|
||||
});
|
||||
}
|
||||
|
||||
buckets.push(ObsBucketReplicationStatsSnapshot {
|
||||
bucket,
|
||||
total_failed_bytes,
|
||||
total_failed_count,
|
||||
last_min_failed_bytes,
|
||||
last_min_failed_count,
|
||||
last_hour_failed_bytes,
|
||||
last_hour_failed_count,
|
||||
sent_bytes,
|
||||
sent_count,
|
||||
for bucket in bucket_names {
|
||||
let bucket_stats = all_bucket_stats.get(&bucket);
|
||||
let proxy = if let Some(stats) = &stats {
|
||||
stats.get_proxy_stats(&bucket).await
|
||||
} else {
|
||||
Default::default()
|
||||
};
|
||||
let proxy = ObsBucketReplicationProxySnapshot {
|
||||
proxied_get_requests_total: i64_to_u64_floor_zero(proxy.get_total),
|
||||
proxied_get_requests_failures: i64_to_u64_floor_zero(proxy.get_failed),
|
||||
proxied_head_requests_total: i64_to_u64_floor_zero(proxy.head_total),
|
||||
@@ -167,13 +237,67 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
|
||||
proxied_get_tagging_requests_failures: i64_to_u64_floor_zero(proxy.get_tag_failed),
|
||||
proxied_delete_tagging_requests_total: i64_to_u64_floor_zero(proxy.delete_tag_total),
|
||||
proxied_delete_tagging_requests_failures: i64_to_u64_floor_zero(proxy.delete_tag_failed),
|
||||
resync_started_count: i64_to_u64_floor_zero(bucket_stats.resync_started_count),
|
||||
resync_completed_count: i64_to_u64_floor_zero(bucket_stats.resync_completed_count),
|
||||
resync_failed_count: i64_to_u64_floor_zero(bucket_stats.resync_failed_count),
|
||||
resync_canceled_count: i64_to_u64_floor_zero(bucket_stats.resync_canceled_count),
|
||||
resync_duration_ms: i64_to_u64_floor_zero(bucket_stats.resync_duration_ms),
|
||||
targets,
|
||||
});
|
||||
};
|
||||
let mut runtime = ObsBucketReplicationRuntimeSnapshot {
|
||||
targets: Vec::with_capacity(bucket_stats.map(|stats| stats.stats.len()).unwrap_or(0)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if let Some(bucket_stats) = bucket_stats {
|
||||
for (target_arn, target_stats) in &bucket_stats.stats {
|
||||
runtime.total_failed_bytes = runtime
|
||||
.total_failed_bytes
|
||||
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
|
||||
runtime.total_failed_count = runtime
|
||||
.total_failed_count
|
||||
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
|
||||
|
||||
let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
|
||||
runtime.last_min_failed_bytes = runtime
|
||||
.last_min_failed_bytes
|
||||
.saturating_add(i64_to_u64_floor_zero(last_min.size));
|
||||
runtime.last_min_failed_count = runtime
|
||||
.last_min_failed_count
|
||||
.saturating_add(i64_to_u64_floor_zero(last_min.count));
|
||||
|
||||
let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
|
||||
runtime.last_hour_failed_bytes = runtime
|
||||
.last_hour_failed_bytes
|
||||
.saturating_add(i64_to_u64_floor_zero(last_hour.size));
|
||||
runtime.last_hour_failed_count = runtime
|
||||
.last_hour_failed_count
|
||||
.saturating_add(i64_to_u64_floor_zero(last_hour.count));
|
||||
|
||||
runtime.sent_bytes = runtime
|
||||
.sent_bytes
|
||||
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
|
||||
runtime.sent_count = runtime
|
||||
.sent_count
|
||||
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
|
||||
|
||||
runtime.targets.push(ObsBucketReplicationTargetStatsSnapshot {
|
||||
target_arn: target_arn.clone(),
|
||||
bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
|
||||
current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
|
||||
latency_ms: target_stats.latency.curr,
|
||||
});
|
||||
}
|
||||
runtime.resync_started_count = i64_to_u64_floor_zero(bucket_stats.resync_started_count);
|
||||
runtime.resync_completed_count = i64_to_u64_floor_zero(bucket_stats.resync_completed_count);
|
||||
runtime.resync_failed_count = i64_to_u64_floor_zero(bucket_stats.resync_failed_count);
|
||||
runtime.resync_canceled_count = i64_to_u64_floor_zero(bucket_stats.resync_canceled_count);
|
||||
runtime.resync_duration_ms = i64_to_u64_floor_zero(bucket_stats.resync_duration_ms);
|
||||
runtime.current_backlog_count = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.count);
|
||||
runtime.current_backlog_bytes = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.bytes);
|
||||
}
|
||||
let durable_bucket = durable_buckets.get(&bucket).cloned().unwrap_or_default();
|
||||
buckets.push(bucket_replication_stats_snapshot_from_parts(
|
||||
bucket,
|
||||
runtime,
|
||||
proxy,
|
||||
durable_mrf_available,
|
||||
durable_bucket,
|
||||
));
|
||||
}
|
||||
|
||||
buckets
|
||||
@@ -243,15 +367,97 @@ mod tests {
|
||||
fn replication_backlog_count_floors_negative_failed_and_queue_values() {
|
||||
assert_eq!(replication_backlog_count([-3, 4].into_iter(), -2), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_backlog_count_keeps_legacy_failed_backlog_semantics() {
|
||||
assert_eq!(replication_backlog_count([9].into_iter(), 0), 9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_replication_snapshot_maps_runtime_and_durable_backlog() {
|
||||
let snapshot = bucket_replication_stats_snapshot_from_parts(
|
||||
"runtime-bucket".to_string(),
|
||||
ObsBucketReplicationRuntimeSnapshot {
|
||||
current_backlog_count: 3,
|
||||
current_backlog_bytes: 4096,
|
||||
resync_failed_count: 2,
|
||||
..Default::default()
|
||||
},
|
||||
ObsBucketReplicationProxySnapshot {
|
||||
proxied_get_requests_total: 7,
|
||||
proxied_get_requests_failures: 1,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
DurableMrfBucketBacklog {
|
||||
bucket: "runtime-bucket".to_string(),
|
||||
count: 5,
|
||||
bytes: 8192,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(snapshot.bucket, "runtime-bucket");
|
||||
assert_eq!(snapshot.current_backlog_count, 3);
|
||||
assert_eq!(snapshot.current_backlog_bytes, 4096);
|
||||
assert!(snapshot.durable_mrf_available);
|
||||
assert_eq!(snapshot.durable_mrf_backlog_count, 5);
|
||||
assert_eq!(snapshot.durable_mrf_backlog_bytes, 8192);
|
||||
assert_eq!(snapshot.resync_failed_count, 2);
|
||||
assert_eq!(snapshot.proxied_get_requests_total, 7);
|
||||
assert_eq!(snapshot.proxied_get_requests_failures, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_replication_snapshot_reports_durable_only_bucket() {
|
||||
let snapshot = bucket_replication_stats_snapshot_from_parts(
|
||||
"durable-only".to_string(),
|
||||
ObsBucketReplicationRuntimeSnapshot::default(),
|
||||
ObsBucketReplicationProxySnapshot::default(),
|
||||
true,
|
||||
DurableMrfBucketBacklog {
|
||||
bucket: "durable-only".to_string(),
|
||||
count: 11,
|
||||
bytes: 2048,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(snapshot.bucket, "durable-only");
|
||||
assert_eq!(snapshot.current_backlog_count, 0);
|
||||
assert!(snapshot.durable_mrf_available);
|
||||
assert_eq!(snapshot.durable_mrf_backlog_count, 11);
|
||||
assert_eq!(snapshot.durable_mrf_backlog_bytes, 2048);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_replication_snapshot_preserves_durable_mrf_unavailable_state() {
|
||||
let snapshot = bucket_replication_stats_snapshot_from_parts(
|
||||
"runtime-only".to_string(),
|
||||
ObsBucketReplicationRuntimeSnapshot {
|
||||
current_backlog_count: 1,
|
||||
current_backlog_bytes: 512,
|
||||
..Default::default()
|
||||
},
|
||||
ObsBucketReplicationProxySnapshot::default(),
|
||||
false,
|
||||
DurableMrfBucketBacklog::default(),
|
||||
);
|
||||
|
||||
assert_eq!(snapshot.current_backlog_count, 1);
|
||||
assert_eq!(snapshot.current_backlog_bytes, 512);
|
||||
assert!(!snapshot.durable_mrf_available);
|
||||
assert_eq!(snapshot.durable_mrf_backlog_count, 0);
|
||||
assert_eq!(snapshot.durable_mrf_backlog_bytes, 0);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod metrics {
|
||||
pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, StorageAdminApi};
|
||||
|
||||
pub(crate) use super::{
|
||||
ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsStore, obs_bucket_replication_stats_snapshot, obs_expiry_state_handle,
|
||||
obs_get_global_bucket_monitor, obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free,
|
||||
obs_is_disk_compression_enabled, obs_load_compression_total_from_memory, obs_load_data_usage_from_backend,
|
||||
obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, obs_transition_state_handle,
|
||||
ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore,
|
||||
obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config,
|
||||
obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
|
||||
obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
|
||||
obs_resolve_object_store_handle, obs_transition_state_handle,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -18,7 +18,26 @@
|
||||
//! used across different logging backends (stdout, file, OpenTelemetry).
|
||||
|
||||
use std::env;
|
||||
use tracing_subscriber::{EnvFilter, filter::LevelFilter};
|
||||
use tracing::Metadata;
|
||||
use tracing_subscriber::{
|
||||
EnvFilter,
|
||||
filter::{FilterFn, LevelFilter, filter_fn},
|
||||
};
|
||||
|
||||
/// Pyroscope emits raw reqwest errors from its background session manager.
|
||||
/// Those errors can include the configured endpoint, so they must never reach
|
||||
/// a RustFS logging sink even when an operator enables verbose dependency logs.
|
||||
pub(super) fn is_pyroscope_dependency_target(target: &str) -> bool {
|
||||
target == "pyroscope" || target.starts_with("pyroscope::") || target.starts_with("Pyroscope::")
|
||||
}
|
||||
|
||||
fn allows_non_pyroscope_dependency_event(metadata: &Metadata<'_>) -> bool {
|
||||
!is_pyroscope_dependency_target(metadata.target())
|
||||
}
|
||||
|
||||
pub(super) fn pyroscope_log_filter() -> FilterFn {
|
||||
filter_fn(allows_non_pyroscope_dependency_event)
|
||||
}
|
||||
|
||||
/// Build an `EnvFilter` from the given log level string.
|
||||
///
|
||||
|
||||
@@ -36,7 +36,7 @@ use crate::cleaner::LogCleaner;
|
||||
use crate::cleaner::types::{CompressionAlgorithm, FileMatchMode};
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::{METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL, METRIC_LOG_CLEANER_RUNS_TOTAL, set_observability_metric_enabled};
|
||||
use crate::telemetry::filter::build_env_filter;
|
||||
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter};
|
||||
use crate::telemetry::rolling::{RollingAppender, Rotation};
|
||||
use metrics::counter;
|
||||
use rustfs_config::observability::{
|
||||
@@ -319,6 +319,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(pyroscope_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
.with(fmt_layer)
|
||||
.try_init()
|
||||
@@ -431,6 +432,7 @@ fn init_file_logging_internal(
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(pyroscope_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
.with(file_layer)
|
||||
.with(stdout_layer)
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
use crate::cleaner::types::FileMatchMode;
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::set_observability_metric_enabled;
|
||||
use crate::telemetry::filter::build_env_filter;
|
||||
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter};
|
||||
use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
|
||||
use crate::telemetry::local::{build_json_log_layer, spawn_cleanup_task};
|
||||
use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
|
||||
@@ -68,7 +68,7 @@ use std::{fs, io::IsTerminal, time::Duration};
|
||||
use tracing::{info, warn};
|
||||
use tracing_error::ErrorLayer;
|
||||
use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
|
||||
use tracing_subscriber::{Layer, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt};
|
||||
use tracing_subscriber::{fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt};
|
||||
|
||||
const GET_OBJECT_DURATION_HISTOGRAM_METRICS: &[&str] = &[
|
||||
"rustfs_io_get_object_request_duration_seconds",
|
||||
@@ -83,6 +83,26 @@ const GET_OBJECT_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
||||
0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
|
||||
];
|
||||
|
||||
#[cfg(all(
|
||||
feature = "pyroscope",
|
||||
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
|
||||
))]
|
||||
const REDACTED_PROFILING_ENDPOINT: &str = "[redacted]";
|
||||
|
||||
#[cfg(all(
|
||||
feature = "pyroscope",
|
||||
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
|
||||
))]
|
||||
fn log_profiler_failure(result: &'static str, error_kind: &'static str) {
|
||||
warn!(
|
||||
backend = "pyroscope",
|
||||
endpoint = REDACTED_PROFILING_ENDPOINT,
|
||||
result,
|
||||
error_kind,
|
||||
"Profiling export agent initialization failed"
|
||||
);
|
||||
}
|
||||
|
||||
/// Initialize the full OpenTelemetry HTTP pipeline (traces + metrics + logs).
|
||||
///
|
||||
/// This function is invoked when at least one OTLP endpoint has been
|
||||
@@ -158,9 +178,7 @@ pub(super) fn init_observability_http(
|
||||
logger_provider = build_logger_provider(&log_ep, config, res, use_stdout)?;
|
||||
|
||||
// Build bridge to capture `tracing` events.
|
||||
otel_bridge = logger_provider
|
||||
.as_ref()
|
||||
.map(|p| OpenTelemetryTracingBridge::new(p).with_filter(build_env_filter(logger_level, None)));
|
||||
otel_bridge = logger_provider.as_ref().map(OpenTelemetryTracingBridge::new);
|
||||
|
||||
// No separate formatting layer is added here; when OTLP logging is
|
||||
// active, the OpenTelemetry bridge is the authoritative sink for
|
||||
@@ -256,6 +274,7 @@ pub(super) fn init_observability_http(
|
||||
let filter = build_env_filter(logger_level, None);
|
||||
tracing_subscriber::registry()
|
||||
.with(filter)
|
||||
.with(pyroscope_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
.with(file_layer_opt)
|
||||
.with(stdout_layer_opt)
|
||||
@@ -531,6 +550,11 @@ pub(super) fn init_profiler(config: &OtelConfig) -> Option<ProfilingAgent> {
|
||||
return None;
|
||||
};
|
||||
|
||||
if url::Url::parse(endpoint).is_err() {
|
||||
log_profiler_failure("profiling_endpoint_invalid", "invalid_endpoint");
|
||||
return None;
|
||||
}
|
||||
|
||||
// Configure Pyroscope Agent
|
||||
let backend = pprof_backend(PprofConfig::default(), BackendConfig::default());
|
||||
let service_name = config.service_name.as_deref().unwrap_or(APP_NAME);
|
||||
@@ -542,28 +566,16 @@ pub(super) fn init_profiler(config: &OtelConfig) -> Option<ProfilingAgent> {
|
||||
.build()
|
||||
{
|
||||
Ok(agent) => agent,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
backend = "pyroscope",
|
||||
endpoint,
|
||||
result = "profiling_agent_build_failed",
|
||||
error = %err,
|
||||
"Profiling export agent initialization failed"
|
||||
);
|
||||
Err(_) => {
|
||||
log_profiler_failure("profiling_agent_build_failed", "build");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
match agent.start() {
|
||||
Ok(agent) => Some(agent),
|
||||
Err(err) => {
|
||||
warn!(
|
||||
backend = "pyroscope",
|
||||
endpoint,
|
||||
result = "profiling_agent_start_failed",
|
||||
error = ?err,
|
||||
"Profiling export agent failed to start"
|
||||
);
|
||||
Err(_) => {
|
||||
log_profiler_failure("profiling_agent_start_failed", "start");
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -650,6 +662,34 @@ fn resolve_signal_timeout(common_timeout_millis: Option<u64>, signal_timeout_mil
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::env;
|
||||
use std::io::{self, Write};
|
||||
use std::process::Command;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
const LOG_TRACER_CHILD_ENV: &str = "RUSTFS_OBS_LOG_TRACER_CHILD";
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestWriter(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
impl Write for TestWriter {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.0.lock().expect("lock test log buffer").extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for TestWriter {
|
||||
type Writer = Self;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// Valid ratios should produce trace-id-ratio sampling.
|
||||
@@ -766,4 +806,107 @@ mod tests {
|
||||
assert_eq!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.first(), Some(&0.0001));
|
||||
assert_eq!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.last(), Some(&10.0));
|
||||
}
|
||||
|
||||
#[cfg(all(
|
||||
feature = "pyroscope",
|
||||
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
|
||||
))]
|
||||
#[test]
|
||||
fn test_init_profiler_invalid_endpoint_redacts_sensitive_components() {
|
||||
let endpoint = "https://profile-user:profile-token@10.24.0.5:invalid/private/profiles?access_token=query-secret";
|
||||
let buffer = Arc::new(Mutex::new(Vec::new()));
|
||||
let writer = TestWriter(Arc::clone(&buffer));
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.with_writer(writer)
|
||||
.finish();
|
||||
|
||||
let config = OtelConfig {
|
||||
profiling_export_enabled: Some(true),
|
||||
profiling_endpoint: Some(endpoint.to_string()),
|
||||
..OtelConfig::default()
|
||||
};
|
||||
|
||||
tracing::subscriber::with_default(subscriber, || assert!(init_profiler(&config).is_none()));
|
||||
|
||||
let rendered = String::from_utf8(buffer.lock().expect("lock test log buffer").clone()).expect("decode test log");
|
||||
assert!(rendered.contains(REDACTED_PROFILING_ENDPOINT));
|
||||
assert!(rendered.contains("error_kind=\"invalid_endpoint\""));
|
||||
for leaked in [
|
||||
"profile-user",
|
||||
"profile-token",
|
||||
"10.24.0.5",
|
||||
"private/profiles",
|
||||
"query-secret",
|
||||
] {
|
||||
assert!(!rendered.contains(leaked), "profiling log leaked {leaked}: {rendered}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pyroscope_upload_failure_targets_stay_filtered_when_env_filter_enables_them() {
|
||||
let endpoint = "https://profile-user:profile-token@10.24.0.5/private/profiles?access_token=query-secret";
|
||||
let buffer = Arc::new(Mutex::new(Vec::new()));
|
||||
let writer = TestWriter(Arc::clone(&buffer));
|
||||
let env_filter = tracing_subscriber::EnvFilter::new("pyroscope=trace")
|
||||
.add_directive("trace".parse().expect("parse trace filter directive"))
|
||||
.add_directive("Pyroscope::Session=trace".parse().expect("parse Pyroscope filter directive"));
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(pyroscope_log_filter())
|
||||
.with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.with_writer(writer),
|
||||
);
|
||||
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
tracing::error!(target: "pyroscope::session", "SessionManager - Failed to send session: {endpoint}");
|
||||
tracing::error!(target: "Pyroscope::Session", "SessionManager - Failed to send session: {endpoint}");
|
||||
});
|
||||
|
||||
let rendered = String::from_utf8(buffer.lock().expect("lock test log buffer").clone()).expect("decode test log");
|
||||
assert!(rendered.is_empty(), "filtered Pyroscope upload failure reached a log sink: {rendered}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pyroscope_log_facade_upload_failure_is_filtered() {
|
||||
let endpoint = "https://profile-user:profile-token@10.24.0.5/private/profiles?access_token=query-secret";
|
||||
if env::var_os(LOG_TRACER_CHILD_ENV).is_some() {
|
||||
tracing_subscriber::registry()
|
||||
.with(build_env_filter("info", None))
|
||||
.with(pyroscope_log_filter())
|
||||
.with(tracing_subscriber::fmt::layer().with_ansi(false).without_time())
|
||||
.try_init()
|
||||
.expect("install isolated LogTracer subscriber");
|
||||
log::error!(target: "pyroscope::session", "SessionManager - Failed to send session: {endpoint}");
|
||||
log::error!(target: "Pyroscope::Session", "SessionManager - Failed to send session: {endpoint}");
|
||||
return;
|
||||
}
|
||||
|
||||
let output = Command::new(env::current_exe().expect("resolve test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"telemetry::otel::tests::test_pyroscope_log_facade_upload_failure_is_filtered",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(LOG_TRACER_CHILD_ENV, "1")
|
||||
.env("RUST_LOG", "trace,pyroscope=trace,Pyroscope::Session=trace")
|
||||
.output()
|
||||
.expect("run isolated LogTracer test process");
|
||||
assert!(output.status.success(), "isolated LogTracer test failed: {output:?}");
|
||||
|
||||
let rendered = format!("{}{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr));
|
||||
for leaked in [
|
||||
"profile-user",
|
||||
"profile-token",
|
||||
"10.24.0.5",
|
||||
"private/profiles",
|
||||
"query-secret",
|
||||
] {
|
||||
assert!(!rendered.contains(leaked), "LogTracer path leaked {leaked}: {rendered}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,7 +32,6 @@ workspace = true
|
||||
default = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/reqwest-0-13",
|
||||
"rustfs-config/hotpath",
|
||||
"rustfs-credentials/hotpath",
|
||||
"rustfs-crypto/hotpath",
|
||||
@@ -80,6 +79,7 @@ pollster.workspace = true
|
||||
proptest = "1"
|
||||
test-case.workspace = true
|
||||
temp-env = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["fmt"] }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
+215
-14
@@ -36,15 +36,10 @@ pub fn is_configured() -> bool {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AuthZPlugin {
|
||||
client: OpaHttpClient,
|
||||
client: reqwest::Client,
|
||||
args: Args,
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
type OpaHttpClient = hotpath::wrap::reqwest::Client;
|
||||
#[cfg(not(feature = "hotpath"))]
|
||||
type OpaHttpClient = reqwest::Client;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum OpaConfigError {
|
||||
#[error("Missing required env var: {0}")]
|
||||
@@ -63,6 +58,44 @@ pub enum OpaConfigError {
|
||||
Connection(reqwest::Error),
|
||||
}
|
||||
|
||||
impl OpaConfigError {
|
||||
pub fn kind(&self) -> &'static str {
|
||||
match self {
|
||||
Self::MissingRequiredEnv(_) => "missing_required_env",
|
||||
Self::InvalidEnvVars(_) => "invalid_env_vars",
|
||||
Self::EnvRead { .. } => "env_read_failed",
|
||||
Self::InvalidStatus(_) => "invalid_status",
|
||||
Self::Connection(_) => "connection_failed",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn redact_opa_connection_error(error: reqwest::Error) -> OpaConfigError {
|
||||
OpaConfigError::Connection(error.without_url())
|
||||
}
|
||||
|
||||
fn opa_request_error_kind(error: &reqwest::Error) -> &'static str {
|
||||
if error.is_timeout() {
|
||||
"timeout"
|
||||
} else if error.is_connect() {
|
||||
"connect"
|
||||
} else if error.is_request() {
|
||||
"request"
|
||||
} else if error.is_body() {
|
||||
"body"
|
||||
} else if error.is_decode() {
|
||||
"decode"
|
||||
} else if error.is_status() {
|
||||
"status"
|
||||
} else if error.is_builder() {
|
||||
"builder"
|
||||
} else if error.is_redirect() {
|
||||
"redirect"
|
||||
} else {
|
||||
"other"
|
||||
}
|
||||
}
|
||||
|
||||
fn check() -> Result<(), OpaConfigError> {
|
||||
let env_list = env::vars();
|
||||
let mut candidate = HashMap::new();
|
||||
@@ -90,7 +123,7 @@ async fn validate(config: &Args) -> Result<(), OpaConfigError> {
|
||||
.timeout(Duration::from_secs(5))
|
||||
.connect_timeout(Duration::from_secs(1))
|
||||
.build()
|
||||
.map_err(OpaConfigError::Connection)?;
|
||||
.map_err(redact_opa_connection_error)?;
|
||||
|
||||
let mut request = client.post(&config.url);
|
||||
if !config.auth_token.is_empty() {
|
||||
@@ -109,7 +142,7 @@ async fn validate(config: &Args) -> Result<(), OpaConfigError> {
|
||||
};
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(OpaConfigError::Connection(err));
|
||||
return Err(redact_opa_connection_error(err));
|
||||
}
|
||||
};
|
||||
Ok(())
|
||||
@@ -155,9 +188,6 @@ impl AuthZPlugin {
|
||||
reqwest::Client::new()
|
||||
});
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
let client = hotpath::http!(client, label = "Policy::OPA");
|
||||
|
||||
Self { client, args: config }
|
||||
}
|
||||
|
||||
@@ -173,7 +203,13 @@ impl AuthZPlugin {
|
||||
Ok(resp) => {
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
error!("OPA returned non-success status: {}", status);
|
||||
error!(
|
||||
component = "policy",
|
||||
subsystem = "opa",
|
||||
result = "request_rejected",
|
||||
status = %status,
|
||||
"OPA request rejected"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -183,13 +219,25 @@ impl AuthZPlugin {
|
||||
OpaResponseEnum::AllowResult(response) => response.result.allow,
|
||||
},
|
||||
Err(err) => {
|
||||
error!("Error parsing OPA response: {:?}", err);
|
||||
error!(
|
||||
component = "policy",
|
||||
subsystem = "opa",
|
||||
result = "response_decode_failed",
|
||||
error_kind = opa_request_error_kind(&err),
|
||||
"OPA response decoding failed"
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Error sending request to OPA: {:?}", err);
|
||||
error!(
|
||||
component = "policy",
|
||||
subsystem = "opa",
|
||||
result = "request_failed",
|
||||
error_kind = opa_request_error_kind(&err),
|
||||
"OPA request failed"
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -258,8 +306,35 @@ enum OpaResponseEnum {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use temp_env;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestWriter(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
impl Write for TestWriter {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
self.0.lock().expect("lock test log buffer").extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for TestWriter {
|
||||
type Writer = Self;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_reqwest_client(_: &reqwest::Client) {}
|
||||
|
||||
#[test]
|
||||
fn test_check_valid_config() {
|
||||
// Use temp_env to temporarily set environment variables
|
||||
@@ -345,4 +420,130 @@ mod tests {
|
||||
};
|
||||
assert!(!args_disabled.enable());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sensitive_opa_endpoints_never_use_hotpath_http_wrapper() {
|
||||
let endpoints = [
|
||||
"https://profile-user:profile-token@10.24.0.5/private/opa?access_token=query-secret",
|
||||
"https://service.internal.example/private/path?token=another-secret",
|
||||
];
|
||||
|
||||
for endpoint in endpoints {
|
||||
let plugin = AuthZPlugin::new(Args {
|
||||
url: endpoint.to_string(),
|
||||
auth_token: "opa-auth-token".to_string(),
|
||||
});
|
||||
|
||||
assert_reqwest_client(&plugin.client);
|
||||
assert_eq!(plugin.args.url, endpoint);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_opa_connection_error_removes_sensitive_endpoint() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind local test listener");
|
||||
let port = listener.local_addr().expect("read local test listener address").port();
|
||||
let server = std::thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().expect("accept local test request");
|
||||
let mut request = [0_u8; 1024];
|
||||
let read = stream.read(&mut request).expect("read local test request");
|
||||
assert!(read > 0, "local test client must send an HTTP request");
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.expect("write local test response");
|
||||
});
|
||||
let endpoint = format!("http://profile-user:profile-token@127.0.0.1:{port}/private/opa?access_token=query-secret");
|
||||
let runtime = tokio::runtime::Runtime::new().expect("build tokio test runtime");
|
||||
let error = runtime.block_on(async {
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("build local test client")
|
||||
.get(&endpoint)
|
||||
.send()
|
||||
.await
|
||||
.expect("send local test request")
|
||||
.error_for_status()
|
||||
.expect_err("test response should be an HTTP error")
|
||||
});
|
||||
server.join().expect("join local test server");
|
||||
assert!(error.url().is_some(), "fixture must carry the endpoint before redaction");
|
||||
|
||||
let OpaConfigError::Connection(error) = redact_opa_connection_error(error) else {
|
||||
panic!("expected a redacted OPA connection error");
|
||||
};
|
||||
let rendered = error.to_string();
|
||||
assert!(error.url().is_none());
|
||||
for leaked in ["profile-user", "profile-token", "127.0.0.1", "private/opa", "query-secret"] {
|
||||
assert!(!rendered.contains(leaked), "redacted error leaked {leaked}: {rendered}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_allowed_rejection_log_redacts_sensitive_endpoint() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind local test listener");
|
||||
let port = listener.local_addr().expect("read local test listener address").port();
|
||||
let server = std::thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().expect("accept local test request");
|
||||
let mut request = [0_u8; 1024];
|
||||
let read = stream.read(&mut request).expect("read local test request");
|
||||
assert!(read > 0, "local test client must send an HTTP request");
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.expect("write local test response");
|
||||
});
|
||||
let endpoint = format!("http://opa-user:opa-token@127.0.0.1:{port}/private/allow?access_token=query-secret");
|
||||
let plugin = AuthZPlugin::new(Args {
|
||||
url: endpoint,
|
||||
auth_token: "authorization-secret".to_string(),
|
||||
});
|
||||
let groups = None;
|
||||
let conditions = HashMap::new();
|
||||
let claims = HashMap::new();
|
||||
let args = PArgs {
|
||||
account: "account",
|
||||
groups: &groups,
|
||||
action: crate::policy::action::Action::None,
|
||||
bucket: "bucket",
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
object: "object",
|
||||
claims: &claims,
|
||||
deny_only: false,
|
||||
};
|
||||
let buffer = Arc::new(Mutex::new(Vec::new()));
|
||||
let writer = TestWriter(Arc::clone(&buffer));
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.with_writer(writer)
|
||||
.finish();
|
||||
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build tokio test runtime");
|
||||
assert!(!runtime.block_on(plugin.is_allowed(&args)));
|
||||
});
|
||||
server.join().expect("join local test server");
|
||||
|
||||
let rendered = String::from_utf8(buffer.lock().expect("lock test log buffer").clone()).expect("decode test log");
|
||||
assert!(!rendered.is_empty(), "OPA request failure did not reach the test log sink");
|
||||
assert!(
|
||||
rendered.contains("request_rejected"),
|
||||
"OPA request failure did not retain its stable result category"
|
||||
);
|
||||
assert!(rendered.contains("500 Internal Server Error"));
|
||||
for leaked in [
|
||||
"opa-user",
|
||||
"opa-token",
|
||||
"127.0.0.1",
|
||||
"private/allow",
|
||||
"query-secret",
|
||||
"authorization-secret",
|
||||
] {
|
||||
assert!(!rendered.contains(leaked), "OPA request log leaked {leaked}: {rendered}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -241,6 +241,16 @@ impl InQueueStats {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn add_current(&self, bytes: i64, count: i64) {
|
||||
self.now_bytes.fetch_add(bytes.max(0), Ordering::Relaxed);
|
||||
self.now_count.fetch_add(count.max(0), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn subtract_current(&self, bytes: i64, count: i64) {
|
||||
saturating_atomic_sub(&self.now_bytes, bytes.max(0));
|
||||
saturating_atomic_sub(&self.now_count, count.max(0));
|
||||
}
|
||||
|
||||
pub fn get_current_bytes(&self) -> i64 {
|
||||
self.now_bytes.load(Ordering::Relaxed)
|
||||
}
|
||||
@@ -250,6 +260,14 @@ impl InQueueStats {
|
||||
}
|
||||
}
|
||||
|
||||
fn saturating_atomic_sub(value: &AtomicI64, delta: i64) {
|
||||
if delta == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let _ = value.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(delta).max(0)));
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct InQueueMetric {
|
||||
pub curr: InQueueStats,
|
||||
@@ -359,6 +377,18 @@ impl QueueCache {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn inc(&mut self, bucket: &str, size: i64) {
|
||||
let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
|
||||
stats.curr.add_current(size, 1);
|
||||
self.sr_queue_stats.curr.add_current(size, 1);
|
||||
}
|
||||
|
||||
pub fn dec(&mut self, bucket: &str, size: i64) {
|
||||
let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
|
||||
stats.curr.subtract_current(size, 1);
|
||||
self.sr_queue_stats.curr.subtract_current(size, 1);
|
||||
}
|
||||
|
||||
pub fn update(&mut self) {
|
||||
let observed_at = Instant::now();
|
||||
self.sr_queue_stats.observe(observed_at);
|
||||
@@ -808,12 +838,10 @@ mod tests {
|
||||
#[test]
|
||||
fn in_queue_metric_observe_updates_rolling_stats() {
|
||||
let mut metric = InQueueMetric::default();
|
||||
metric.curr.now_bytes.store(128, Ordering::Relaxed);
|
||||
metric.curr.now_count.store(4, Ordering::Relaxed);
|
||||
metric.curr.add_current(128, 4);
|
||||
metric.observe(Instant::now());
|
||||
|
||||
metric.curr.now_bytes.store(256, Ordering::Relaxed);
|
||||
metric.curr.now_count.store(6, Ordering::Relaxed);
|
||||
metric.curr.add_current(128, 2);
|
||||
metric.observe(Instant::now());
|
||||
|
||||
assert_eq!(metric.curr.bytes, 256);
|
||||
@@ -824,6 +852,21 @@ mod tests {
|
||||
assert_eq!(metric.last_minute.count, 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queue_cache_decrement_saturates_at_zero() {
|
||||
let mut cache = QueueCache::new();
|
||||
cache.inc("bucket", 128);
|
||||
cache.dec("bucket", 512);
|
||||
cache.dec("bucket", 1);
|
||||
|
||||
let bucket = cache.get_bucket_stats("bucket");
|
||||
let site = cache.get_site_stats();
|
||||
assert_eq!(bucket.curr.count, 0);
|
||||
assert_eq!(bucket.curr.bytes, 0);
|
||||
assert_eq!(site.curr.count, 0);
|
||||
assert_eq!(site.curr.bytes, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fail_stats_recent_since_tracks_windows() {
|
||||
let mut stats = FailStats::default();
|
||||
|
||||
@@ -43,6 +43,9 @@ allow-git = [
|
||||
# Presigned expiry and constant-time authentication fixes pending upstream merge.
|
||||
# owner: cxymds review: 2026-10
|
||||
"https://github.com/cxymds/s3s.git",
|
||||
# MiMalloc fork pinned for hotpath allocation counting support.
|
||||
# owner: houseme review: 2026-10
|
||||
"https://github.com/xonatius/mimalloc_rust.git",
|
||||
"https://github.com/apache/datafusion.git",
|
||||
]
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ For how the Vault backends authenticate (static token, AppRole, Vault Agent toke
|
||||
| Static | `Static` | Provided out-of-band via environment/file; never persisted by RustFS | Operator-managed secret distribution | No state persisted by RustFS | Rejected (read-only backend) | Simple deployments with an external secret manager |
|
||||
| Vault KV2 | `VaultKV2` (legacy alias `Vault`) | Stored **directly** in Vault KV v2 (Base64-encoded plaintext) | Vault ACLs + KV v2 at-rest encryption + TLS only | Delegated to Vault storage | Versioned retention (immutable per-version records + current pointer) | Deployments that accept Vault KV ACLs as the sole confidentiality boundary |
|
||||
| Vault Transit | `VaultTransit` | Key-encryption keys never leave Vault; only Transit ciphertext is visible outside | Vault Transit engine (cryptographic isolation) | Delegated to Vault storage | Via Vault Transit key versioning | Deployments that need key material to be unreadable through storage APIs |
|
||||
| AWS KMS | `AWS` (alias `AwsKms`) | Key material never leaves AWS KMS; RustFS mirrors no key state | AWS KMS (cryptographic isolation) + IAM | Delegated to AWS | On-demand `RotateKeyOnDemand`; prior backing keys stay usable for decryption | Deployments already rooted in AWS IAM that want AWS as the cryptographic root — read [AWS KMS: deviations from the shared backend contract](#aws-kms-deviations-from-the-shared-backend-contract) first |
|
||||
|
||||
## Vault KV2: what the backend does and does not do
|
||||
|
||||
@@ -44,7 +45,7 @@ path "secret/metadata/rustfs/kms/keys/*" {
|
||||
Notes:
|
||||
|
||||
- The trailing wildcards also cover the per-version material records that rotation creates under `.../keys/{key_id}/versions/{N}`; no extra policy paths are needed.
|
||||
- `delete` on the metadata path is required for permanent key deletion (`force_immediate`); drop it if you never hard-delete keys. RustFS refuses `force_immediate` unless the server sets `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, so leaving that gate off keeps the capability unreachable no matter what the Vault policy allows.
|
||||
- `delete` on the metadata path is required for permanent key deletion (`force_immediate`); drop it if you never hard-delete keys.
|
||||
- Do not attach `sudo`, wildcard mounts, or Transit paths to this policy; the KV2 backend does not use them.
|
||||
- Auditing KV reads on the key prefix is strongly recommended: every read event is a potential master-key disclosure.
|
||||
|
||||
@@ -62,7 +63,7 @@ Decryption loads exactly the version recorded in the envelope and fails closed w
|
||||
|
||||
- Every version record that any stored DEK envelope references must remain readable. Until an object rewrap/migration capability exists, assume **every** version of a rotated key is referenced: destroying a version record permanently orphans all objects whose DEKs it wrapped.
|
||||
- Version records are ordinary KV v2 secrets under the key subtree. Never run `kv metadata delete` or `kv destroy` against `{prefix}/{key_id}/versions/*`, and do not apply `delete-version-after` or retention tooling to that subtree. RustFS-managed retention does not rely on KV2's own secret versioning (each version record has a single KV revision), so KV `max-versions` settings do not protect or endanger history — but metadata deletion always removes a record entirely.
|
||||
- Permanent key deletion through RustFS (`force_immediate` after `PendingDeletion`) purges the key's version records together with the key record; that is the only supported way to remove them. It is refused by default: the server must set `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, and the request must echo the key id back as `confirm_key_id`. Leave the gate off unless you are actively destroying keys, and turn it off again afterwards — the pending-deletion window plus `CancelKeyDeletion` is the only recovery path for objects encrypted under the key.
|
||||
- Permanent key deletion through RustFS (`force_immediate` after `PendingDeletion`) purges the key's version records together with the key record; that is the only supported way to remove them.
|
||||
- For Vault Transit, retention is governed by the Transit key's `min_decryption_version`: never raise it above the oldest version that may still protect live ciphertext.
|
||||
|
||||
### Upgrade before first rotation (hard constraint)
|
||||
@@ -146,6 +147,25 @@ Use **Vault Transit** (`VaultTransit`) when key material must be cryptographical
|
||||
|
||||
Use **Vault KV2** only when you accept that the Vault ACL on the key path *is* the confidentiality boundary and you want the operational simplicity of a single KV mount.
|
||||
|
||||
## AWS KMS: deviations from the shared backend contract
|
||||
|
||||
Select it with `RUSTFS_KMS_BACKEND=aws`. Credentials and region resolution are delegated entirely to the standard `aws-config` provider chain (environment, shared profile, container/IMDS role), so RustFS never stores, persists, or redacts AWS credential material of its own. Only two non-credential settings are read: `RUSTFS_KMS_AWS_REGION` and `RUSTFS_KMS_AWS_ENDPOINT_URL`. A plaintext (`http://`) endpoint override would expose every KMS request including plaintext data keys, so it is refused unless the development opt-in is set.
|
||||
|
||||
AWS owns key state, backing-key rotation, and the deletion window, and this backend mirrors none of it locally. That makes four behaviours differ from every RustFS-managed backend. Verify each against your operational assumptions before switching:
|
||||
|
||||
| Behaviour | RustFS-managed backends | AWS KMS backend |
|
||||
| --- | --- | --- |
|
||||
| Decryption with a `Disabled` or `PendingDeletion` key | Kept working, so disabling a key never breaks reads of objects already encrypted under it | **Refused by AWS.** Objects encrypted under a key that is later disabled become unreadable until it is re-enabled |
|
||||
| Key deletion | Physical deletion available | **No physical delete.** `ScheduleKeyDeletion` is the only removal path; AWS destroys the material when the 7-30 day window elapses. RustFS never destroys AWS-held material, and `force_immediate` is refused |
|
||||
| Cancelling a scheduled deletion | Key returns to `Enabled` | Key is left **`Disabled`**; enable it explicitly to make it usable again |
|
||||
| Creating a key under a caller-chosen name | The requested name becomes the key id | **Refused.** AWS assigns identifiers and this backend does not manage aliases, so a named create would produce a key unreachable by that name |
|
||||
|
||||
Two consequences follow from that last row: **SSE-S3 key auto-creation and the synthetic KMS probe are unavailable on this backend**, because both address a key by a name they choose. Pre-create keys in AWS and reference them by AWS key id or ARN.
|
||||
|
||||
Key versions are opaque. AWS addresses backing keys internally and picks the right one to decrypt with, so RustFS reports `key_version` as 1 and cannot enumerate versions. Rotation uses `RotateKeyOnDemand`, which retains prior backing keys for decryption; AWS's separate automatic yearly rotation is neither enabled nor reported on by RustFS.
|
||||
|
||||
The AWS backend is not configurable through the KMS admin API — use the environment variables above at startup.
|
||||
|
||||
## Local backend durability and deployment support matrix
|
||||
|
||||
The Local backend stores one JSON record per key (`<key_id>.key`) plus an Argon2id salt file (`.master-key.salt`) inside the configured `key_dir`. This section documents which deployments that layout supports and how the backend recovers from a crash or power loss. For where the key material lives and who can read it, see the [backend comparison](#backend-comparison) above.
|
||||
|
||||
+1
-1
@@ -330,7 +330,7 @@ mimalloc = { workspace = true }
|
||||
libsystemd.workspace = true
|
||||
|
||||
[target.'cfg(not(target_os = "windows"))'.dependencies]
|
||||
libmimalloc-sys = { version = "0.1.49", features = ["extended"] }
|
||||
libmimalloc-sys.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "macro-diagnostics"] }
|
||||
|
||||
@@ -76,6 +76,8 @@ fn existing_vault_auth(config: &KmsConfig) -> Option<rustfs_kms::config::VaultAu
|
||||
rustfs_kms::config::BackendConfig::VaultTransit(vault) => Some(vault.auth_method.clone()),
|
||||
rustfs_kms::config::BackendConfig::Local(_) => None,
|
||||
rustfs_kms::config::BackendConfig::Static(_) => None,
|
||||
// AWS credentials come from the aws-config chain, not from KMS config.
|
||||
rustfs_kms::config::BackendConfig::Aws(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -189,9 +191,6 @@ async fn save_kms_config(config: &KmsConfig) -> Result<(), String> {
|
||||
|
||||
fn decode_persisted_kms_config(data: &[u8]) -> serde_json::Result<(KmsConfig, bool)> {
|
||||
let mut config: KmsConfig = serde_json::from_slice(data)?;
|
||||
// The immediate-deletion gate is per-server operator state, never stored,
|
||||
// so a config loaded from cluster storage still has to pick it up here.
|
||||
config.allow_immediate_deletion = rustfs_kms::config::allow_immediate_deletion_from_env();
|
||||
let value: serde_json::Value = serde_json::from_slice(data)?;
|
||||
let is_missing_development_flag = value
|
||||
.as_object()
|
||||
|
||||
@@ -1092,9 +1092,6 @@ impl Operation for DeleteKmsKeyHandler {
|
||||
key_id: request.key_id.clone(),
|
||||
pending_window_in_days: request.pending_window_in_days,
|
||||
force_immediate: request.force_immediate,
|
||||
// The endpoint does not accept a confirmation yet, so an immediate
|
||||
// deletion asked for here is always refused by the service gate.
|
||||
confirm_key_id: None,
|
||||
};
|
||||
|
||||
match manager.delete_key(kms_request).await {
|
||||
|
||||
@@ -53,6 +53,7 @@ fn backend_name(backend: &KmsBackend) -> &'static str {
|
||||
KmsBackend::VaultKv2 => "vault-kv2",
|
||||
KmsBackend::VaultTransit => "vault-transit",
|
||||
KmsBackend::Static => "static",
|
||||
KmsBackend::Aws => "aws",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -291,7 +291,6 @@ fn build_local_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::c
|
||||
file_permissions: Some(0o600),
|
||||
}),
|
||||
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
||||
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
||||
default_key_id: cfg.kms_default_key_id.clone(),
|
||||
timeout: std::time::Duration::from_secs(30),
|
||||
retry_attempts: 3,
|
||||
@@ -329,7 +328,6 @@ fn build_vault_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::c
|
||||
tls: None,
|
||||
})),
|
||||
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
||||
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
||||
default_key_id: cfg.kms_default_key_id.clone(),
|
||||
timeout: std::time::Duration::from_secs(30),
|
||||
retry_attempts: 3,
|
||||
@@ -365,7 +363,6 @@ fn build_vault_transit_kms_config(cfg: &config::Config) -> std::io::Result<rustf
|
||||
..rustfs_kms::config::VaultTransitConfig::default()
|
||||
})),
|
||||
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
||||
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
||||
default_key_id: cfg.kms_default_key_id.clone(),
|
||||
timeout: std::time::Duration::from_secs(30),
|
||||
retry_attempts: 3,
|
||||
@@ -420,7 +417,6 @@ fn build_static_kms_config(cfg: &config::Config) -> std::io::Result<rustfs_kms::
|
||||
default_key_id: cfg.kms_default_key_id.clone().or(Some(key_id)),
|
||||
backend_config: rustfs_kms::config::BackendConfig::Static(static_config),
|
||||
allow_insecure_dev_defaults: cfg.kms_allow_insecure_dev_defaults,
|
||||
allow_immediate_deletion: rustfs_kms::config::allow_immediate_deletion_from_env(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
+56
-8
@@ -17,27 +17,37 @@ use std::alloc::{GlobalAlloc, Layout};
|
||||
|
||||
#[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))]
|
||||
#[derive(Default)]
|
||||
struct DefaultMiMalloc;
|
||||
struct MiMallocAllocator;
|
||||
|
||||
#[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))]
|
||||
// SAFETY: allocation and deallocation are forwarded unchanged to MiMalloc, so
|
||||
// SAFETY: allocation operations are forwarded unchanged to MiMalloc, so
|
||||
// MiMalloc's GlobalAlloc guarantees apply to every returned pointer and layout.
|
||||
#[allow(unsafe_code)]
|
||||
unsafe impl GlobalAlloc for DefaultMiMalloc {
|
||||
unsafe impl GlobalAlloc for MiMallocAllocator {
|
||||
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
|
||||
// SAFETY: the caller upholds GlobalAlloc's contract for layout.
|
||||
unsafe { mimalloc::MiMalloc.alloc(layout) }
|
||||
}
|
||||
|
||||
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
|
||||
// SAFETY: the caller upholds GlobalAlloc's contract for layout.
|
||||
unsafe { mimalloc::MiMalloc.alloc_zeroed(layout) }
|
||||
}
|
||||
|
||||
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
|
||||
// SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
|
||||
unsafe { mimalloc::MiMalloc.dealloc(ptr, layout) }
|
||||
}
|
||||
|
||||
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
|
||||
// SAFETY: ptr and layout came from this allocator and are forwarded unchanged.
|
||||
unsafe { mimalloc::MiMalloc.realloc(ptr, layout, new_size) }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "hotpath", feature = "hotpath-alloc"))]
|
||||
#[global_allocator]
|
||||
static GLOBAL: hotpath::CountingAllocator<DefaultMiMalloc> = hotpath::CountingAllocator::new();
|
||||
static GLOBAL: hotpath::CountingAllocator<MiMallocAllocator> = hotpath::CountingAllocator::new();
|
||||
|
||||
#[cfg(not(all(feature = "hotpath", feature = "hotpath-alloc")))]
|
||||
#[global_allocator]
|
||||
@@ -49,14 +59,52 @@ fn main() {
|
||||
rustfs::startup_entrypoint::run_process();
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "hotpath", feature = "hotpath-alloc"))]
|
||||
#[cfg(all(test, feature = "hotpath", feature = "hotpath-alloc", not(target_os = "windows")))]
|
||||
mod tests {
|
||||
#[test]
|
||||
// SAFETY: This test inspects a live allocation pointer with mimalloc's heap
|
||||
// ownership API without dereferencing or extending the pointer lifetime.
|
||||
#[allow(unsafe_code)]
|
||||
fn hotpath_allocator_uses_mimalloc() {
|
||||
let allocation = Box::new([0_u8; 64]);
|
||||
fn hotpath_allocation_workload_uses_mimalloc() {
|
||||
let _guard = hotpath::MeasurementGuardSync::new("rustfs::tests::hotpath_allocation_workload_uses_mimalloc", false, false);
|
||||
let mut allocation = Vec::with_capacity(64);
|
||||
allocation.extend_from_slice(&[7_u8; 64]);
|
||||
|
||||
// SAFETY: the live Box pointer is valid to inspect for heap ownership.
|
||||
assert_eq!(allocation.len(), 64);
|
||||
// SAFETY: the live Vec pointer is valid to inspect for heap ownership.
|
||||
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(allocation.as_ptr().cast()) });
|
||||
}
|
||||
|
||||
#[test]
|
||||
// SAFETY: This test directly exercises the allocator wrapper and releases
|
||||
// every successful allocation with the matching layout.
|
||||
#[allow(unsafe_code)]
|
||||
fn mimalloc_allocator_forwards_extended_global_alloc_operations() {
|
||||
use std::alloc::{GlobalAlloc, Layout};
|
||||
|
||||
let layout = Layout::from_size_align(32, 8).expect("valid test allocation layout");
|
||||
let grown_layout = Layout::from_size_align(64, 8).expect("valid grown test allocation layout");
|
||||
let allocator = super::MiMallocAllocator;
|
||||
|
||||
// SAFETY: The pointer is checked for null before use and later released
|
||||
// through the same allocator with the corresponding layout.
|
||||
let ptr = unsafe { allocator.alloc_zeroed(layout) };
|
||||
assert!(!ptr.is_null());
|
||||
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(ptr.cast()) });
|
||||
assert!(unsafe { std::slice::from_raw_parts(ptr, 32).iter().all(|byte| *byte == 0) });
|
||||
|
||||
// SAFETY: `ptr` was allocated by `allocator` with `layout`; on failure
|
||||
// the original allocation remains valid and is released below.
|
||||
let grown_ptr = unsafe { allocator.realloc(ptr, layout, 64) };
|
||||
if grown_ptr.is_null() {
|
||||
// SAFETY: `ptr` is still valid when realloc returns null.
|
||||
unsafe { allocator.dealloc(ptr, layout) };
|
||||
panic!("mimalloc realloc failed in allocator smoke test");
|
||||
}
|
||||
|
||||
assert!(unsafe { libmimalloc_sys::mi_is_in_heap_region(grown_ptr.cast()) });
|
||||
// SAFETY: `grown_ptr` was reallocated by `allocator` and is released
|
||||
// with the matching grown layout.
|
||||
unsafe { allocator.dealloc(grown_ptr, grown_layout) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,14 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::app::context;
|
||||
use rustfs_config::ENV_RUSTFS_BROWSER_REDIRECT_URL;
|
||||
use rustfs_iam::federation::FederatedIdentityService;
|
||||
use std::sync::Arc;
|
||||
use tracing::warn;
|
||||
|
||||
const LOG_COMPONENT_MAIN: &str = "main";
|
||||
const LOG_SUBSYSTEM_AUTH: &str = "auth";
|
||||
const EVENT_OIDC_BROWSER_REDIRECT_FALLBACK: &str = "oidc_browser_redirect_fallback";
|
||||
|
||||
pub(crate) use context::{
|
||||
AppContext, NotifyInterface, ServerContextSlot, default_notify_interface as fallback_notify_interface,
|
||||
@@ -22,9 +29,9 @@ pub(crate) use context::{
|
||||
default_s3select_db_interface as fallback_s3select_db_interface,
|
||||
default_scanner_metrics_interface as fallback_scanner_metrics_interface,
|
||||
default_server_config_interface as fallback_server_config_interface,
|
||||
default_storage_class_interface as fallback_storage_class_interface, publish_federated_identity_service,
|
||||
publish_server_config, publish_storage_class_config, resolve_action_credentials as current_action_credentials,
|
||||
resolve_boot_time as current_boot_time, resolve_bucket_metadata_handle as current_bucket_metadata_handle,
|
||||
default_storage_class_interface as fallback_storage_class_interface, publish_server_config, publish_storage_class_config,
|
||||
resolve_action_credentials as current_action_credentials, resolve_boot_time as current_boot_time,
|
||||
resolve_bucket_metadata_handle as current_bucket_metadata_handle,
|
||||
resolve_bucket_monitor_handle as current_bucket_monitor_handle, resolve_buffer_config as current_buffer_config,
|
||||
resolve_daily_tier_stats as current_daily_tier_stats, resolve_deployment_id as current_deployment_id,
|
||||
resolve_encryption_service as current_encryption_service, resolve_endpoints_handle as current_endpoints_handle,
|
||||
@@ -52,6 +59,24 @@ pub(crate) use context::{
|
||||
resolve_tier_config_handle as current_tier_config_handle, resolve_token_signing_key as current_token_signing_key,
|
||||
};
|
||||
|
||||
pub(crate) fn publish_federated_identity_service(service: Arc<FederatedIdentityService>) -> bool {
|
||||
let browser_redirect_url = rustfs_utils::get_env_opt_str(ENV_RUSTFS_BROWSER_REDIRECT_URL);
|
||||
if service.has_providers() && browser_redirect_url.as_deref().is_none_or(|value| value.trim().is_empty()) {
|
||||
warn!(
|
||||
event = EVENT_OIDC_BROWSER_REDIRECT_FALLBACK,
|
||||
component = LOG_COMPONENT_MAIN,
|
||||
subsystem = LOG_SUBSYSTEM_AUTH,
|
||||
state = "fallback_enabled",
|
||||
configuration = ENV_RUSTFS_BROWSER_REDIRECT_URL,
|
||||
fallback_source = "request_host_and_forwarded_proto",
|
||||
reason = "trusted_console_origin_not_configured",
|
||||
"OIDC browser redirect fallback enabled"
|
||||
);
|
||||
}
|
||||
|
||||
context::publish_federated_identity_service(service)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_test_outbound_tls_generation(generation: u64) {
|
||||
context::set_test_outbound_tls_generation(generation);
|
||||
@@ -68,3 +93,136 @@ pub(crate) use context::{IamInterface, KmsInterface, NotificationSystemInterface
|
||||
pub(crate) fn current_app_context() -> Option<Arc<AppContext>> {
|
||||
context::get_global_app_context()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_iam::{
|
||||
federation::{
|
||||
FederatedAuthorization, FederatedCodeExchange, FederatedIdentityProvider, FederatedIdentityRegistry,
|
||||
Result as FederationResult,
|
||||
},
|
||||
oidc::{OidcProviderConfig, OidcProviderSummary},
|
||||
};
|
||||
use std::{
|
||||
io::{self, Write},
|
||||
sync::Mutex,
|
||||
};
|
||||
use tracing_subscriber::{Registry, fmt::MakeWriter, layer::SubscriberExt};
|
||||
|
||||
struct TestProvider {
|
||||
has_providers: bool,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl FederatedIdentityProvider for TestProvider {
|
||||
fn has_providers(&self) -> bool {
|
||||
self.has_providers
|
||||
}
|
||||
|
||||
fn list_providers(&self) -> Vec<OidcProviderSummary> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn list_visible_providers(&self) -> Vec<OidcProviderSummary> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn provider_config(&self, _id: &str) -> Option<&OidcProviderConfig> {
|
||||
None
|
||||
}
|
||||
|
||||
async fn authorize_url(
|
||||
&self,
|
||||
_provider_id: &str,
|
||||
_redirect_uri: &str,
|
||||
_redirect_after: Option<String>,
|
||||
) -> FederationResult<String> {
|
||||
unreachable!("startup publication test does not authorize")
|
||||
}
|
||||
|
||||
async fn exchange_code(&self, _state: &str, _code: &str, _redirect_uri: &str) -> FederationResult<FederatedCodeExchange> {
|
||||
unreachable!("startup publication test does not exchange codes")
|
||||
}
|
||||
|
||||
async fn verify_web_identity_token(&self, _jwt: &str) -> FederationResult<FederatedAuthorization> {
|
||||
unreachable!("startup publication test does not verify tokens")
|
||||
}
|
||||
|
||||
async fn create_logout_token(&self, _provider_id: &str, _id_token: &str) -> FederationResult<String> {
|
||||
unreachable!("startup publication test does not create logout tokens")
|
||||
}
|
||||
|
||||
async fn build_logout_url(
|
||||
&self,
|
||||
_logout_token: &str,
|
||||
_post_logout_redirect_uri: &str,
|
||||
) -> FederationResult<Option<String>> {
|
||||
unreachable!("startup publication test does not build logout URLs")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CapturedLogs(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
struct CapturedLogWriter(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
impl Write for CapturedLogWriter {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.0.lock().expect("captured log lock").extend_from_slice(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'writer> MakeWriter<'writer> for CapturedLogs {
|
||||
type Writer = CapturedLogWriter;
|
||||
|
||||
fn make_writer(&'writer self) -> Self::Writer {
|
||||
CapturedLogWriter(self.0.clone())
|
||||
}
|
||||
}
|
||||
|
||||
fn test_service(has_providers: bool) -> Arc<FederatedIdentityService> {
|
||||
let provider = Arc::new(TestProvider { has_providers });
|
||||
Arc::new(FederatedIdentityService::new(FederatedIdentityRegistry::new(provider)))
|
||||
}
|
||||
|
||||
fn capture_startup_publication(has_providers: bool, browser_redirect_url: Option<&str>) -> String {
|
||||
temp_env::with_var(ENV_RUSTFS_BROWSER_REDIRECT_URL, browser_redirect_url, || {
|
||||
let logs = CapturedLogs::default();
|
||||
let captured = logs.0.clone();
|
||||
let subscriber = Registry::default().with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.without_time()
|
||||
.with_target(false)
|
||||
.with_level(true)
|
||||
.with_ansi(false)
|
||||
.with_writer(logs),
|
||||
);
|
||||
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
assert!(publish_federated_identity_service(test_service(has_providers)));
|
||||
});
|
||||
|
||||
String::from_utf8(captured.lock().expect("captured log lock").clone()).expect("captured logs must be UTF-8")
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_startup_publication_warns_only_for_request_header_fallback() {
|
||||
let output = capture_startup_publication(true, None);
|
||||
|
||||
assert!(output.contains("WARN"), "{output}");
|
||||
assert!(output.contains(EVENT_OIDC_BROWSER_REDIRECT_FALLBACK), "{output}");
|
||||
assert!(output.contains(ENV_RUSTFS_BROWSER_REDIRECT_URL), "{output}");
|
||||
assert!(output.contains("request_host_and_forwarded_proto"), "{output}");
|
||||
assert!(output.contains("trusted_console_origin_not_configured"), "{output}");
|
||||
assert!(capture_startup_publication(false, None).is_empty());
|
||||
assert!(capture_startup_publication(true, Some("https://console.example.com")).is_empty());
|
||||
assert!(!capture_startup_publication(true, Some(" ")).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
Executable
+77
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env bash
|
||||
# Assert that self-hosted runners are ephemeral: one job per runner, ever.
|
||||
#
|
||||
# WHY THIS IS CHECKED CONTINUOUSLY RATHER THAN ONCE
|
||||
#
|
||||
# rustfs/rustfs is public, and pull_request jobs run on self-hosted runners —
|
||||
# they execute the PR's own build.rs, proc-macros and tests, which is arbitrary
|
||||
# code. What keeps that from reaching the release build is that each runner is a
|
||||
# fresh ARC pod that handles exactly one job and is destroyed. Nothing in this
|
||||
# repository enforces it: it is a property of the ARC scale-set configuration,
|
||||
# which lives outside the repo and can be changed without any PR. So it is
|
||||
# asserted from the outside, on real data.
|
||||
#
|
||||
# A repeated runner_name means a runner served two jobs, i.e. state survived
|
||||
# between them, i.e. a poisoned PR job could leave something behind for whatever
|
||||
# runs next — including, while the release build shares the sm-standard-2 pool,
|
||||
# a release build.
|
||||
#
|
||||
# Usage: scripts/ci/check_runner_ephemerality.sh [run-count]
|
||||
# run-count defaults to 40. MIN_SAMPLE (default 10) is the number of observed
|
||||
# assignments below which the window is reported INCONCLUSIVE rather than OK.
|
||||
#
|
||||
# Exit codes: 0 ephemeral, 1 a runner served two jobs, 2 inconclusive or broken.
|
||||
set -euo pipefail
|
||||
|
||||
REPO="${GITHUB_REPOSITORY:-rustfs/rustfs}"
|
||||
LIMIT="${1:-40}"
|
||||
# Below this many observed assignments the window says nothing useful: with the
|
||||
# pool saturated, most sm-* jobs sit queued with runner_name still null, and a
|
||||
# handful of samples would pass by coincidence rather than by evidence.
|
||||
MIN_SAMPLE="${MIN_SAMPLE:-10}"
|
||||
|
||||
command -v gh >/dev/null || { echo "ERROR: gh CLI not found" >&2; exit 2; }
|
||||
|
||||
runs="$(gh run list --repo "$REPO" --limit "$LIMIT" --json databaseId --jq '.[].databaseId')"
|
||||
[ -n "$runs" ] || { echo "ERROR: no runs returned for $REPO" >&2; exit 2; }
|
||||
|
||||
names="$(
|
||||
for run in $runs; do
|
||||
gh api "repos/${REPO}/actions/runs/${run}/jobs" --paginate \
|
||||
--jq '.jobs[] | select(.runner_name != null) | select(.runner_name | startswith("sm-")) | .runner_name' 2>/dev/null || true
|
||||
done
|
||||
)"
|
||||
|
||||
total="$(printf '%s\n' "$names" | grep -c . || true)"
|
||||
if [ "${total:-0}" -eq 0 ]; then
|
||||
echo "ERROR: no self-hosted (sm-*) runner names found across $LIMIT runs." >&2
|
||||
echo " Either the label scheme changed or the API shape did — this check" >&2
|
||||
echo " must not silently pass by finding nothing." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [ "$total" -lt "$MIN_SAMPLE" ]; then
|
||||
echo "INCONCLUSIVE: only ${total} self-hosted job assignments across ${LIMIT} runs" >&2
|
||||
echo " (need ${MIN_SAMPLE}). Most sm-* jobs are probably still queued, so" >&2
|
||||
echo " runner_name is null. Re-run with a larger window when the pool drains." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
dupes="$(printf '%s\n' "$names" | sort | uniq -d)"
|
||||
|
||||
if [ -n "$dupes" ]; then
|
||||
echo "ERROR: self-hosted runners served more than one job — not ephemeral:" >&2
|
||||
printf '%s\n' "$dupes" | while read -r name; do
|
||||
count="$(printf '%s\n' "$names" | grep -c "^${name}$")"
|
||||
echo " ${name}: ${count} jobs" >&2
|
||||
done
|
||||
echo "" >&2
|
||||
echo "State can survive between jobs on those runners. Because this is a" >&2
|
||||
echo "public repository whose pull_request jobs run PR-authored code, and" >&2
|
||||
echo "because build.yml's release legs share the sm-standard-2 pool, a" >&2
|
||||
echo "poisoned PR job could reach a release build. Check the ARC scale-set" >&2
|
||||
echo "configuration (rustfs/backlog#1602)." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: ${total} self-hosted job assignments across ${LIMIT} runs, all on distinct runners"
|
||||
Reference in New Issue
Block a user