Compare commits

..

2 Commits

Author SHA1 Message Date
overtrue 0d30b623a7 feat(kms): orchestrate Vault restore ordering, verification, and mismatch detection
Restoring a Vault-backed deployment is an operator action for the trust
root and a RustFS action for everything else. This adds the orchestration
around that split.

Ordering is structural, not documented: VaultRestoreSequence refuses any
stage that is not the next one, so material and metadata can never be
published before the Transit/HSM trust root is verified back.

Every way the recovered Vault can disagree with the bundle is a distinct
typed mismatch: a missing Transit key, a min_decryption_version raised
above what the bundle still needs, a Transit version history restored
below the snapshot point, a rolled-back KV generation, and cluster,
namespace, or mount drift. A dry-run reports them all and issues reads
only; an actual restore refuses on the first one, before any KV path is
probed.

Writes are create-only check-and-set throughout, so version regression,
generation rollback, and reviving deleted state are impossible rather
than merely rejected, and a lost race stops the restore instead of
overwriting. A reserved KV record is the single commit point: published
before the first write and removed after the last, it names exactly the
records this restore owns, so re-running the same bundle rolls forward
idempotently and abort takes back precisely what was written — never a
record that pre-existed. Every interruption converges to the complete old
or the complete new state.

Tests drive the whole surface offline through the scripted Vault
responder, including the zero-write contract asserted at the wire; the
live-Vault drill is ignored but kept compiling.

Refs rustfs/backlog#1572 (part of rustfs/backlog#1562)
2026-08-01 12:03:11 +08:00
overtrue 6900e67d32 feat(kms): record external Vault references in the backup manifest
A Vault-backed bundle never carries its cryptographic root: Transit keys
are non-exportable and KV2 records live in Vault's own storage. Record
instead which external state the bundle was captured against, so a
restore can prove the recovered Vault is the right one: cluster identity,
namespace, KV mount and path prefix, the Transit key with the version
window the bundle still needs, and the per-key KV generation.

The reference schema is required exactly for Vault backends, rejected for
the others, and its field set is pinned by a test so no credential-
carrying field can be added by accident.

Also split the bundle manifest reader so consumers of non-Local bundles
share the same decode path, and widen three artifact-framing items to
pub(crate) so producer and consumer cannot drift on the AEAD framing.

Refs rustfs/backlog#1572 (part of rustfs/backlog#1562)
2026-08-01 12:02:58 +08:00
125 changed files with 3174 additions and 11405 deletions
+4
View File
@@ -300,6 +300,9 @@ path = "junit.xml"
# negative-path siblings of each family stay in as regression guards.
# * rustfs#4843 — over-limit archive entry paths hard-reject the whole
# archive even under ignore-errors semantics.
# * rustfs#4846 — distributed-lock quorum tests misclassify as timeout
# under parallel load (multi-node in-process clusters; natural home is
# ci-7's nightly cluster lane).
[profile.e2e-full]
default-filter = """
package(e2e_test)
@@ -308,6 +311,7 @@ default-filter = """
& !test(/^replication_extension_test::/)
& !test(/^multipart_auth_test::test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled$/)
& !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(ignores_invalid_entries_when_requested|supports_standard_headers_with_combined_extract_options)$/)
& !test(/^reliant::lock::test_distributed_lock_(2_nodes_grpc_read_survives_failed_node|4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes)$/)
"""
fail-fast = false
+1 -12
View File
@@ -111,18 +111,7 @@ runs:
- name: Setup Rust cache
uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
# false is rust-cache's own default. With true, cleanup.ts returns
# *before* pruning ~/.cargo/registry/src, and config.ts archives the
# whole registry — so every cache carried the unpacked source tree of
# every dependency, not just "a few extra crates".
#
# No coverage is lost: getPackages runs `cargo metadata --all-features`,
# a strict superset of any single lane's feature closure, and -sys crates
# are explicitly exempted from pruning (their src timestamps would
# otherwise trigger rebuilds). Anything pruned is re-unpacked from the
# .crate files still in registry/cache, whose mtimes crates.io
# normalises, so cargo fingerprints stay valid.
cache-all-crates: false
cache-all-crates: true
cache-on-failure: true
shared-key: ${{ inputs.cache-shared-key }}
save-if: ${{ inputs.cache-save-if }}
@@ -49,8 +49,6 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Install ripgrep
run: |
+1 -14
View File
@@ -80,8 +80,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
# cargo-deny compiles nothing, so the full setup composite (apt packages,
# protoc, flatc, nextest, rustfmt/clippy) was pure overhead here. It does
@@ -100,9 +98,7 @@ jobs:
- name: Setup Rust cache
uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
# Same reasoning as the setup composite: true archives every
# dependency's unpacked source tree.
cache-all-crates: false
cache-all-crates: true
cache-on-failure: true
shared-key: rustfs-cargo-deny
save-if: ${{ github.ref == 'refs/heads/main' }}
@@ -123,8 +119,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Report unpinned GitHub Actions
run: ./scripts/security/check_workflow_pins.sh --enforce
@@ -135,9 +129,6 @@ jobs:
- name: Check every job declares a timeout
run: ./scripts/security/check_job_timeouts.sh
- name: Check checkouts clear their credentials
run: ./scripts/security/check_persist_credentials.sh
- name: Check preview release workflow policy
run: ./scripts/security/check_preview_release_workflow.sh
@@ -152,8 +143,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Dependency Review
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5
@@ -185,8 +174,6 @@ jobs:
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
-6
View File
@@ -99,8 +99,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Determine build strategy
id: check
@@ -259,7 +257,6 @@ jobs:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
fetch-depth: 0
- name: Setup Rust environment
@@ -798,7 +795,6 @@ jobs:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
fetch-depth: 0
- name: Create GitHub Release
@@ -856,8 +852,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Download all build artifacts
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
-50
View File
@@ -65,14 +65,6 @@ on:
- ".github/workflows/docker.yml"
- ".github/workflows/audit.yml"
workflow_dispatch:
inputs:
emit_timings:
description: >-
Also emit cargo --timings for the ci-dev build and upload it. Used to
decide whether sccache is worth adopting (rustfs/backlog#1601 gate).
required: false
default: false
type: boolean
permissions:
contents: read
@@ -107,33 +99,6 @@ jobs:
cache-save-if: 'true'
install-build-packaging-tools: 'false'
# rustfs/backlog#1601 gate. sccache can only cache compilation units whose
# --emit includes link, so it covers workspace rlibs and nothing else:
# clippy is metadata-only, and the ~100 test binaries, the rustfs bin and
# every build script invoke the system linker. Before spending a bucket,
# credentials and a supply-chain boundary on it, measure how much of the
# build is actually rlib codegen.
#
# Read from the report: workspace lib codegen as a share of the build, and
# s3select-query's own rlib as a share. The plan adopts sccache only above
# 50% and 25% respectively; if linking dominates instead, the answer is
# mold/lld plus split-debuginfo, which is exactly the part sccache cannot
# touch. Off by default — this doubles the ci-dev build.
- name: Build ci-dev superset (with --timings)
if: inputs.emit_timings
env:
CARGO_BUILD_JOBS: "2"
run: cargo build --workspace --all-targets --timings
- name: Upload cargo timings report
if: inputs.emit_timings
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: cargo-timings-ci-dev
path: target/cargo-timings/
retention-days: 30
if-no-files-found: error
# --all-targets covers the test binaries nextest builds, including
# e2e_test, which test-and-lint's own run excludes. The second build adds
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
@@ -148,21 +113,6 @@ jobs:
cargo build --workspace --all-targets
cargo build -p rustfs --bins --features e2e-test-hooks
# Runs before rust-cache's post step, so these are the sizes it is about
# to archive. Reported so the cache-all-crates decision stays evidence-led:
# registry/src is what that flag prunes, registry/cache is what the pruned
# sources are re-unpacked from. See rustfs/backlog#1600.
- name: Report cache input sizes
if: always()
run: |
{
echo "### Cache input sizes (ci-dev)"
echo '```'
du -sh ~/.cargo/registry/src ~/.cargo/registry/cache ~/.cargo/registry/index \
~/.cargo/git target 2>/dev/null || true
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
# Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2.
warm-ci-feat-rio:
name: Warm ci-feat-rio
-4
View File
@@ -79,8 +79,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Install ripgrep
run: sudo apt-get update && sudo apt-get install -y ripgrep
@@ -127,8 +125,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
# Docs-only PRs skip the full code CI, but they are exactly where a
# planning-type document could be slipped in (git add -f bypasses
-26
View File
@@ -95,8 +95,6 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Typos check with custom config file
uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
@@ -114,8 +112,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Install ripgrep
run: sudo apt-get update && sudo apt-get install -y ripgrep
@@ -357,8 +353,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -397,8 +391,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -441,8 +433,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -471,8 +461,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -504,8 +492,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -546,8 +532,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -610,8 +594,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
# Full setup with dependency caching: the smoke-suite step below
# compiles the e2e_test crate, which pulls in most of the workspace.
@@ -696,8 +678,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -741,8 +721,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Clean up previous test run
run: |
@@ -797,8 +775,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Download debug binary
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -871,8 +847,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Download debug binary
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
+4 -21
View File
@@ -22,18 +22,11 @@ on:
issue_comment:
types: [created, edited]
# Least privilege at the top, widened per job below. This workflow runs on
# pull_request_target and issue_comment, so it holds full secrets on every fork
# PR and on any comment anyone writes — the one place in this repository where a
# compromised action would be handed a repo-write token. It does not check out
# or execute PR code, so there is no pwn-request path today, but the blast
# radius should not depend on that staying true.
#
# contents: write in particular was never used: the signature records are
# written to rustfs/cla through the scoped app token created below, and nothing
# here writes to this repository's contents.
permissions:
contents: read
contents: write
pull-requests: write
issues: write
checks: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}
@@ -43,9 +36,6 @@ jobs:
cancel-closed-pr-runs:
name: Cancel Closed PR Runs
if: github.event_name == 'pull_request_target' && github.event.action == 'closed'
# Echoes one line; the run exists only so the concurrency group cancels the
# in-flight run of a closed PR.
permissions: {}
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
@@ -54,13 +44,6 @@ jobs:
cla:
if: ${{ (github.event_name != 'issue_comment' || github.event.issue.pull_request) && (github.event_name != 'pull_request_target' || github.event.action != 'closed') }}
# checks: write reports the merge-queue check run; pull-requests and issues
# let cla-bot comment and label. contents stays read — see the note above.
permissions:
contents: read
checks: write
issues: write
pull-requests: write
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
-4
View File
@@ -62,8 +62,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -118,8 +116,6 @@ jobs:
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
-3
View File
@@ -98,7 +98,6 @@ jobs:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
# For workflow_run events, checkout the specific commit that triggered the workflow
ref: ${{ github.event.workflow_run.head_sha || github.sha }}
@@ -306,8 +305,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Login to Docker Hub
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
@@ -64,8 +64,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -126,8 +124,6 @@ jobs:
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
-4
View File
@@ -142,8 +142,6 @@ jobs:
TEST_MODE: ${{ matrix.test-mode }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
# Provision Python explicitly rather than trusting the runner image to
# ship a working pip (ci-1: a bare python3 without pip is what broke the
@@ -363,8 +361,6 @@ jobs:
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
-8
View File
@@ -87,8 +87,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -154,8 +152,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Download prebuilt fuzz binaries
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -211,8 +207,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Download prebuilt fuzz binaries
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
@@ -260,8 +254,6 @@ jobs:
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
-4
View File
@@ -50,8 +50,6 @@ jobs:
steps:
- name: Checkout helm chart repo
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
# Both inputs reach the shell through env rather than `${{ }}`
# interpolation. A git ref name may contain `$(...)` — anything without a
@@ -123,8 +121,6 @@ jobs:
- name: Checkout helm package repo
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
# persist-credentials-exempt: this checkout's token IS the push credential —
# the job git-pushes to rustfs/helm below. Clearing it breaks chart publishing.
repository: rustfs/helm
token: ${{ secrets.RUSTFS_HELM_PACKAGE }}
-2
View File
@@ -58,8 +58,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
-4
View File
@@ -125,8 +125,6 @@ jobs:
timeout-minutes: 120
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Enable buildx
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
@@ -272,8 +270,6 @@ jobs:
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
-4
View File
@@ -40,10 +40,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
# persist-credentials-exempt: update-flake-lock pushes the branch and opens
# the PR. It passes FLAKE_UPDATE_TOKEN to create-pull-request itself rather
# than reusing .git/config, but that is unverified — exempt until a
# workflow_dispatch run confirms it (rustfs/backlog#1602).
- name: Install Nix
uses: DeterminateSystems/determinate-nix-action@629b284231c2a82554b724e357e47fc6020833c8 # v3
-2
View File
@@ -71,8 +71,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Install Nix
uses: DeterminateSystems/determinate-nix-action@4eea0b33e3d1f02ecfe37cf16e7204c424009606 # v3.21.0
-5
View File
@@ -99,8 +99,6 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -150,7 +148,6 @@ jobs:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
fetch-depth: 0 # baseline is built from origin/main
- name: Setup Rust environment
@@ -397,8 +394,6 @@ jobs:
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
-81
View File
@@ -1,81 +0,0 @@
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Asserts that the self-hosted runners are still ephemeral — one job per pod.
#
# This repository is public and its pull_request jobs run on those runners,
# executing the PR's own build.rs, proc-macros and tests. The only thing keeping
# that code from reaching a later job is that each ARC pod handles exactly one
# job and is then destroyed. That guarantee lives in the ARC scale-set
# configuration, outside this repository, where it can be changed without any PR
# — so it is asserted here from the outside, against real run data, instead of
# being assumed.
#
# Monthly rather than per-PR: the property changes only when someone
# reconfigures the scale set, and the check costs a few dozen API calls.
# See docs/ci/runners.md and rustfs/backlog#1602.
name: Runner Hygiene
on:
schedule:
- cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
workflow_dispatch:
permissions:
contents: read
concurrency:
group: runner-hygiene
cancel-in-progress: false
jobs:
check-ephemerality:
name: Check runner ephemerality
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
# Exit 2 (inconclusive / broken) is deliberately not a pass: a window
# where every sm-* job was still queued would otherwise look identical to
# a clean bill of health.
- name: Assert one job per self-hosted runner
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: ./scripts/ci/check_runner_ephemerality.sh 40
alert-on-failure:
name: Alert on scheduled failure
needs: [check-ephemerality]
# Same ci-8 mechanism as coverage.yml, audit.yml and the nightly lanes:
# scheduled runs file a tracking issue, manual dispatch stays quiet so
# debugging never produces a spurious alert.
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -63,8 +63,6 @@ jobs:
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
Generated
+36 -150
View File
@@ -290,11 +290,11 @@ dependencies = [
[[package]]
name = "ar_archive_writer"
version = "0.5.3"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6"
checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348"
dependencies = [
"object 0.39.1",
"object 0.37.3",
]
[[package]]
@@ -599,16 +599,6 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "assert-json-diff"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "astral-tokio-tar"
version = "0.6.4"
@@ -953,32 +943,6 @@ dependencies = [
"uuid",
]
[[package]]
name = "aws-sdk-kms"
version = "1.114.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0b7d906608ee41e7ddea9983577ba82200435644d567d63dc34e822e088b453"
dependencies = [
"arc-swap",
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-json",
"aws-smithy-observability",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
"aws-smithy-schema",
"aws-smithy-types",
"aws-types",
"bytes",
"fastrand",
"http 0.2.12",
"http 1.5.0",
"regex-lite",
"tracing",
]
[[package]]
name = "aws-sdk-s3"
version = "1.140.0"
@@ -1194,23 +1158,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e"
dependencies = [
"aws-smithy-async",
"aws-smithy-protocol-test",
"aws-smithy-runtime-api",
"aws-smithy-types",
"bytes",
"h2",
"http 1.5.0",
"http-body 1.1.0",
"hyper",
"hyper-rustls",
"hyper-util",
"indexmap 2.14.0",
"pin-project-lite",
"rustls",
"rustls-native-certs",
"rustls-pki-types",
"serde",
"serde_json",
"tokio",
"tokio-rustls",
"tower",
@@ -1237,25 +1195,6 @@ dependencies = [
"aws-smithy-runtime-api",
]
[[package]]
name = "aws-smithy-protocol-test"
version = "0.64.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f76511a0e223ce78deb6a78b8afebda99cb737cfbc8a58d96dcb190f012dd40a"
dependencies = [
"assert-json-diff",
"aws-smithy-runtime-api",
"base64-simd",
"cbor-diag",
"ciborium",
"http 0.2.12",
"pretty_assertions",
"regex-lite",
"roxmltree",
"serde_json",
"thiserror 2.0.19",
]
[[package]]
name = "aws-smithy-query"
version = "0.62.0"
@@ -1589,7 +1528,7 @@ version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
]
[[package]]
@@ -1608,7 +1547,7 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
]
[[package]]
@@ -1659,7 +1598,7 @@ version = "3.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f"
dependencies = [
"darling 0.23.0",
"darling 0.20.11",
"ident_case",
"prettyplease",
"proc-macro2",
@@ -1740,9 +1679,9 @@ dependencies = [
[[package]]
name = "bytesize"
version = "2.6.0"
version = "2.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493"
checksum = "3d7c8918969267b2932ffd5655509bbbea0833823058c378876953217f5fc50e"
[[package]]
name = "bytestring"
@@ -1819,25 +1758,6 @@ dependencies = [
"cipher 0.5.2",
]
[[package]]
name = "cbor-diag"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429"
dependencies = [
"bs58",
"chrono",
"data-encoding",
"half",
"nom 7.1.3",
"num-bigint",
"num-rational",
"num-traits",
"separator",
"url",
"uuid",
]
[[package]]
name = "cc"
version = "1.4.0"
@@ -1950,7 +1870,7 @@ version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
dependencies = [
"crypto-common 0.1.6",
"crypto-common 0.1.7",
"inout 0.1.4",
]
@@ -1968,9 +1888,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.6.5"
version = "4.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf"
checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
dependencies = [
"clap_builder",
"clap_derive",
@@ -1978,9 +1898,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.6.5"
version = "4.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078"
checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
dependencies = [
"anstream",
"anstyle",
@@ -2408,7 +2328,7 @@ version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
"rand_core 0.6.4",
"subtle",
"zeroize",
@@ -2433,11 +2353,11 @@ dependencies = [
[[package]]
name = "crypto-common"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
"typenum",
]
@@ -3634,7 +3554,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer 0.10.4",
"const-oid 0.9.6",
"crypto-common 0.1.6",
"crypto-common 0.1.7",
"subtle",
]
@@ -3894,7 +3814,7 @@ dependencies = [
"crypto-bigint 0.5.5",
"digest 0.10.7",
"ff 0.13.1",
"generic-array 0.14.9",
"generic-array 0.14.7",
"group 0.13.0",
"hkdf 0.12.4",
"pem-rfc7468 0.7.0",
@@ -4339,9 +4259,9 @@ dependencies = [
[[package]]
name = "generic-array"
version = "0.14.9"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
@@ -4354,7 +4274,7 @@ version = "1.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
dependencies = [
"generic-array 0.14.9",
"generic-array 0.14.7",
"rustversion",
"typenum",
]
@@ -5381,7 +5301,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
dependencies = [
"block-padding 0.3.3",
"generic-array 0.14.9",
"generic-array 0.14.7",
]
[[package]]
@@ -5915,7 +5835,8 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libmimalloc-sys"
version = "0.1.49"
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=1cdadea43e9c5a0f054b65be21200ce580e4eb13#1cdadea43e9c5a0f054b65be21200ce580e4eb13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
dependencies = [
"cc",
"cty",
@@ -6324,7 +6245,8 @@ dependencies = [
[[package]]
name = "mimalloc"
version = "0.1.52"
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=1cdadea43e9c5a0f054b65be21200ce580e4eb13#1cdadea43e9c5a0f054b65be21200ce580e4eb13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
dependencies = [
"libmimalloc-sys",
]
@@ -6746,17 +6668,6 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
dependencies = [
"num-bigint",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -6821,7 +6732,7 @@ version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
dependencies = [
"base64 0.22.1",
"base64 0.21.7",
"chrono",
"getrandom 0.2.17",
"http 1.5.0",
@@ -7946,7 +7857,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
dependencies = [
"heck",
"itertools 0.14.0",
"itertools 0.13.0",
"log",
"multimap",
"once_cell",
@@ -7966,7 +7877,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
dependencies = [
"heck",
"itertools 0.14.0",
"itertools 0.13.0",
"log",
"multimap",
"petgraph 0.8.3",
@@ -7987,7 +7898,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
dependencies = [
"anyhow",
"itertools 0.14.0",
"itertools 0.13.0",
"proc-macro2",
"quote",
"syn 2.0.119",
@@ -8000,7 +7911,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
dependencies = [
"anyhow",
"itertools 0.14.0",
"itertools 0.13.0",
"proc-macro2",
"quote",
"syn 2.0.119",
@@ -8720,15 +8631,6 @@ dependencies = [
"serde",
]
[[package]]
name = "roxmltree"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b"
dependencies = [
"xmlparser",
]
[[package]]
name = "rsa"
version = "0.9.10"
@@ -8822,9 +8724,9 @@ dependencies = [
[[package]]
name = "russh"
version = "0.62.5"
version = "0.62.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da7c230e0ed9cbeb92fbad6c8848985d6df2a1464c0dc247a021abd666e9005e"
checksum = "b8b67b5a0d8068c89dcbe9d95df986af7a851d1f3c604525274c37468e60464f"
dependencies = [
"aes 0.9.2",
"aws-lc-rs",
@@ -9609,16 +9511,10 @@ dependencies = [
"arc-swap",
"argon2",
"async-trait",
"aws-config",
"aws-sdk-kms",
"aws-smithy-http-client",
"aws-smithy-runtime-api",
"aws-smithy-types",
"base64 0.23.0",
"chacha20poly1305",
"hex",
"hotpath",
"http 1.5.0",
"insta",
"jiff",
"md-5 0.11.0",
@@ -9627,7 +9523,6 @@ dependencies = [
"moka",
"rand 0.10.2",
"reqwest",
"rustfs-s3-types",
"rustfs-security-governance",
"rustfs-utils",
"rustify",
@@ -9813,7 +9708,6 @@ dependencies = [
"hotpath",
"jiff",
"libc",
"log",
"metrics",
"num_cpus",
"nvml-wrapper",
@@ -9849,7 +9743,6 @@ dependencies = [
"tracing-error",
"tracing-opentelemetry",
"tracing-subscriber",
"url",
"zstd",
]
@@ -9881,7 +9774,6 @@ dependencies = [
"time",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
@@ -10716,7 +10608,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
dependencies = [
"base16ct 0.2.0",
"der 0.7.10",
"generic-array 0.14.9",
"generic-array 0.14.7",
"pkcs8 0.10.2",
"subtle",
"zeroize",
@@ -10769,12 +10661,6 @@ dependencies = [
"serde_core",
]
[[package]]
name = "separator"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5"
[[package]]
name = "seq-macro"
version = "0.3.6"
@@ -11712,7 +11598,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.3.4",
"getrandom 0.4.3",
"once_cell",
"rustix",
"windows-sys 0.61.2",
+4 -6
View File
@@ -173,7 +173,7 @@ tower-http = { version = "0.7.0" }
# Serialization and Data Formats
apache-avro = "0.21.0"
bytes = { version = "1.12.1" }
bytesize = "2.6.0"
bytesize = "2.4.2"
byteorder = "1.5.0"
flatbuffers = "25.12.19"
form_urlencoded = "1.2.2"
@@ -227,7 +227,6 @@ atoi = "3.1.0"
atomic_enum = "0.3.0"
aws-config = { version = "1.10.1" }
aws-credential-types = { version = "1.3.0" }
aws-sdk-kms = { default-features = false, version = "1.114.0" }
aws-sdk-s3 = { default-features = false, version = "1.140.0" }
aws-sdk-sts = { default-features = false, version = "1.110.0" }
aws-smithy-http-client = { default-features = false, version = "1.2.0" }
@@ -236,7 +235,7 @@ aws-smithy-types = { version = "1.6.1" }
base64 = "0.23.0"
base64-simd = "0.8.0"
brotli = "8.0.4"
clap = { version = "4.6.5" }
clap = { version = "4.6.4" }
const-str = { version = "1.1.0" }
convert_case = "0.11.0"
criterion = { version = "0.8" }
@@ -341,15 +340,14 @@ libunftp = { version = "0.23.0" }
unftp-core = "0.1.0"
suppaftp = { version = "10.0.1" }
rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.62.5" }
russh = { version = "0.62.4" }
russh-sftp = "2.3.0"
# WebDAV
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13" }
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13", features = ["extended"] }
mimalloc = "0.1.52"
hotpath = { version = "0.22.0", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.48" }
-13
View File
@@ -230,19 +230,6 @@ pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
/// Default value: false
pub const DEFAULT_KMS_ENABLE: bool = false;
/// Environment variable enabling per-key KMS authorization on the SSE-KMS data path.
///
/// When enabled, an SSE-KMS write additionally requires `kms:GenerateDataKey` and an
/// SSE-KMS read additionally requires `kms:Decrypt` on the resolved key, evaluated as
/// the requesting identity. SSE-S3 and SSE-C are unaffected.
pub const ENV_RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY: &str = "RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY";
/// Default per-key KMS authorization mode for the SSE-KMS data path.
///
/// Off for now so deployments whose identity policies only grant s3 actions keep
/// working; the default flips to on in a later release.
pub const DEFAULT_KMS_ENFORCE_SSE_KEY_POLICY: bool = false;
/// Environment variable for server KMS backend.
pub const ENV_RUSTFS_KMS_BACKEND: &str = "RUSTFS_KMS_BACKEND";
+60 -1
View File
@@ -26,16 +26,75 @@
//! Later batches tracked on backlog#1154: config get/set, info, pools status,
//! group lifecycle, import/export IAM.
use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::{Client, Config};
use http::header::{CONTENT_TYPE, HOST};
use reqwest::StatusCode;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use serial_test::serial;
use std::error::Error;
use tokio::time::{Duration, sleep};
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
type BoxError = Box<dyn Error + Send + Sync>;
/// Signs and sends an admin HTTP request with the given credential, returning
/// status and body. Native `/rustfs/admin/v3` requests and responses are plain
/// JSON (the MinIO-compat encryption applies only to `/minio/admin/v3` paths).
async fn admin_request(
base_url: &str,
method: http::Method,
path_and_query: &str,
body: Option<String>,
access_key: &str,
secret_key: &str,
) -> Result<(StatusCode, String), BoxError> {
let url = format!("{base_url}{path_and_query}");
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
let mut builder = http::Request::builder()
.method(method.clone())
.uri(uri)
.header(HOST, authority)
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
if body.is_some() {
builder = builder.header(CONTENT_TYPE, "application/json");
}
let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default();
let signed = sign_v4(builder.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
let mut request = local_http_client().request(reqwest_method, &url);
for (name, value) in signed.headers() {
request = request.header(name, value);
}
if let Some(body) = body {
request = request.body(body);
}
let response = request.send().await?;
let status = response.status();
let text = response.text().await.unwrap_or_default();
Ok((status, text))
}
/// Root-credential admin request that must succeed; returns the response body.
async fn admin_ok(
env: &RustFSTestEnvironment,
method: http::Method,
path_and_query: &str,
body: Option<String>,
) -> Result<String, BoxError> {
let (status, text) = admin_request(&env.url, method.clone(), path_and_query, body, &env.access_key, &env.secret_key).await?;
if !status.is_success() {
return Err(format!("{method} {path_and_query} failed: {status} {text}").into());
}
Ok(text)
}
fn build_s3_client(url: &str, access_key: &str, secret_key: &str) -> Client {
let config = Config::builder()
-57
View File
@@ -24,12 +24,7 @@
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::{Client, Config};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use http::header::{CONTENT_TYPE, HOST};
use reqwest::Client as HttpClient;
use reqwest::StatusCode;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use std::ffi::OsStr;
use std::fs as stdfs;
use std::path::{Path, PathBuf};
@@ -80,58 +75,6 @@ pub fn local_http_client() -> HttpClient {
.expect("failed to build local reqwest client")
}
/// Signs and sends an admin HTTP request with the given credentials.
pub(crate) async fn admin_request(
base_url: &str,
method: http::Method,
path_and_query: &str,
body: Option<String>,
access_key: &str,
secret_key: &str,
) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{base_url}{path_and_query}");
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
let mut request = http::Request::builder()
.method(method.clone())
.uri(uri)
.header(HOST, authority)
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
if body.is_some() {
request = request.header(CONTENT_TYPE, "application/json");
}
let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "admin request body is too large")?;
let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
let mut request = local_http_client().request(method, &url);
for (name, value) in signed.headers() {
request = request.header(name, value);
}
if let Some(body) = body {
request = request.body(body);
}
let response = request.send().await?;
let status = response.status();
let body = response.text().await?;
Ok((status, body))
}
/// Sends a root-credential admin request and returns its successful response body.
pub(crate) async fn admin_ok(
env: &RustFSTestEnvironment,
method: http::Method,
path_and_query: &str,
body: Option<String>,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
let (status, response_body) =
admin_request(&env.url, method.clone(), path_and_query, body, &env.access_key, &env.secret_key).await?;
if !status.is_success() {
return Err(format!("{method} {path_and_query} failed: {status} {response_body}").into());
}
Ok(response_body)
}
/// Resolve the RustFS binary relative to the workspace.
pub fn rustfs_binary_path() -> PathBuf {
rustfs_binary_path_with_features(requested_rustfs_build_features().as_deref())
+4 -6
View File
@@ -15,7 +15,9 @@
use super::{grpc_lock_client::GrpcLockClient, grpc_lock_server::spawn_lock_server};
use rustfs_lock::client::{LockClient, local::LocalClient};
use rustfs_lock::{GlobalLockManager, LockInfo, LockRequest, LockResponse, LockStats, LockType, NamespaceLock, ObjectKey};
use rustfs_lock::{
GlobalLockManager, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockType, NamespaceLock, ObjectKey,
};
use std::sync::Arc;
use std::time::Duration;
@@ -33,11 +35,7 @@ struct FailingClient;
#[async_trait::async_trait]
impl rustfs_lock::LockClient for FailingClient {
async fn acquire_lock(&self, _request: &rustfs_lock::LockRequest) -> rustfs_lock::Result<LockResponse> {
// Match RemoteClient's transport-failure response so the coordinator can count this node toward quorum loss.
Ok(LockResponse::failure(
"Remote lock RPC failed: simulated gRPC node failure",
Duration::ZERO,
))
Err(LockError::internal("simulated gRPC node failure"))
}
async fn release(&self, _lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
+46 -18
View File
@@ -382,6 +382,7 @@ struct ManualTransitionRunReport {
skipped_delete_marker: u64,
skipped_directory: u64,
skipped_replication: u64,
skipped_already_transitioned: u64,
skipped_already_in_flight: u64,
skipped_queue_full: u64,
skipped_queue_closed: u64,
@@ -407,6 +408,41 @@ fn assert_completed_or_in_flight_partial(state: &str, report: &ManualTransitionR
}
}
fn assert_conflict_winner_report(state: &str, report: &ManualTransitionRunReport, expected_objects: u64, context: &str) {
assert_completed_or_in_flight_partial(state, report, context);
if report.skipped_already_in_flight > 0 {
assert!(
report.scanned <= expected_objects,
"{context}: scanned more objects than the conflict scope contains: {report:#?}"
);
assert!(
report.eligible <= expected_objects,
"{context}: marked more objects eligible than the conflict scope contains: {report:#?}"
);
assert_eq!(
report.enqueued + report.skipped_already_in_flight,
report.eligible,
"{context}: partial in-flight accounting must cover every eligible object: {report:#?}"
);
} else {
assert_eq!(report.scanned, expected_objects, "{context}: {report:#?}");
assert_eq!(
report.eligible + report.skipped_already_transitioned,
expected_objects,
"{context}: {report:#?}"
);
assert_eq!(
report.enqueued + report.skipped_already_in_flight,
expected_objects,
"{context}: {report:#?}"
);
}
assert_eq!(
report.transition_completed, report.enqueued,
"{context}: winner must wait for all queued transitions: {report:#?}"
);
}
#[derive(Debug, Deserialize)]
struct ManualTransitionQueueSnapshot {
queue_capacity: u64,
@@ -1240,15 +1276,8 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
let mut hot = RustFSTestEnvironment::new().await?;
hot.start_rustfs_server_with_env(
vec![],
&[
("RUSTFS_SCANNER_ENABLED", "false"),
("RUSTFS_SCANNER_CYCLE", "3600"),
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
],
)
.await?;
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
.await?;
let hot_client = hot.create_s3_client();
add_rustfs_tier(&hot, &cold).await?;
@@ -1318,21 +1347,20 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes
assert_eq!(conflict.cancel_endpoint, status_endpoint);
assert!(!conflict.scope_key.is_empty());
manual_transition_job_cancel(&hot, cancel_endpoint).await?;
let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT).await?;
assert_eq!(terminal.job_id, job_id);
assert_eq!(terminal.status, "cancelled", "terminal conflict winner response: {terminal:#?}");
assert!(!terminal.report.dry_run);
assert_eq!(terminal.report.bucket, MANUAL_ASYNC_CONFLICT_BUCKET);
assert_eq!(terminal.report.prefix, accepted.report.prefix);
assert!(terminal.report.cancelled, "terminal conflict winner response: {terminal:#?}");
assert_eq!(terminal.report.scanned, 0, "terminal conflict winner response: {terminal:#?}");
assert_eq!(terminal.report.enqueued, 0, "terminal conflict winner response: {terminal:#?}");
assert_eq!(
terminal.report.transition_completed, 0,
"terminal conflict winner response: {terminal:#?}"
assert_conflict_winner_report(
&terminal.status,
&terminal.report,
MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
"terminal conflict winner response",
);
assert_eq!(terminal.report.dry_run_eligible, 0, "terminal conflict winner response: {terminal:#?}");
assert_eq!(terminal.report.transition_failed, 0, "terminal conflict winner response: {terminal:#?}");
assert_eq!(terminal.report.tier_failure, 0, "terminal conflict winner response: {terminal:#?}");
let after_remote_count = cold_tier_object_count(&cold_client).await?;
assert!(after_remote_count >= before_remote_count);
assert!(after_remote_count <= before_remote_count + MANUAL_ASYNC_CONFLICT_OBJECTS);
+37 -421
View File
@@ -12,35 +12,22 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
use aws_sdk_sts::config::retry::RetryConfig;
use aws_sdk_sts::config::{Credentials, Region};
use aws_sdk_sts::error::ProvideErrorMetadata;
use aws_sdk_sts::operation::RequestId;
use aws_sdk_sts::{Client, Config};
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
use bytes::Bytes;
use http::header::{AUTHORIZATION, CONTENT_TYPE};
use http::{Request, Response};
use http_body_util::{BodyExt, Full};
use hyper::body::Incoming;
use hyper::server::conn::http1;
use hyper::service::service_fn;
use hyper_util::rt::TokioIo;
use serde_json::Value;
use http::header::{CONTENT_TYPE, HOST};
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use serial_test::serial;
use std::collections::BTreeSet;
use std::convert::Infallible;
use std::error::Error;
use std::sync::Arc;
use tokio::net::TcpListener;
use tokio::sync::{Notify, mpsc};
use tokio::task::{JoinHandle, JoinSet};
use tokio::time::{Duration, timeout};
type BoxError = Box<dyn Error + Send + Sync>;
type TestResult = Result<(), BoxError>;
const OPA_AUTH_TOKEN: &str = "sts-opa-token";
fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
let mut config = Config::builder()
@@ -62,14 +49,32 @@ fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Opti
}
async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(String, String), BoxError> {
let body = admin_ok(
env,
http::Method::PUT,
"/rustfs/admin/v3/add-service-accounts",
Some(serde_json::json!({ "targetUser": env.access_key.clone() }).to_string()),
)
.await?;
let response: Value = serde_json::from_str(&body)?;
let path = "/rustfs/admin/v3/add-service-accounts";
let url = format!("{}{path}", env.url);
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
let body = serde_json::json!({ "targetUser": env.access_key.clone() }).to_string();
let request = http::Request::builder()
.method(http::Method::PUT)
.uri(uri)
.header(HOST, authority)
.header(CONTENT_TYPE, "application/json")
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
.body(Body::empty())?;
let content_length = i64::try_from(body.len()).map_err(|_| "service account request body is too large")?;
let signed = sign_v4(request, content_length, &env.access_key, &env.secret_key, "", "us-east-1");
let mut request = local_http_client().put(&url);
for (name, value) in signed.headers() {
request = request.header(name, value);
}
let response = request.body(body).send().await?;
let status = response.status();
let body = response.text().await?;
if !status.is_success() {
return Err(format!("create service account failed: {status} {body}").into());
}
let response: serde_json::Value = serde_json::from_str(&body)?;
let access_key = response["credentials"]["accessKey"]
.as_str()
.ok_or("service account response should contain credentials.accessKey")?
@@ -81,237 +86,28 @@ async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(Str
Ok((access_key, secret_key))
}
async fn create_user_with_policy(
env: &RustFSTestEnvironment,
user: &str,
secret: &str,
policy_name: &str,
statements: Value,
) -> TestResult {
create_user(env, user, secret).await?;
admin_ok(
env,
http::Method::PUT,
&format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
Some(
serde_json::json!({
"Version": "2012-10-17",
"Statement": statements,
})
.to_string(),
),
)
.await?;
admin_ok(
env,
http::Method::POST,
"/rustfs/admin/v3/idp/builtin/policy/attach",
Some(serde_json::json!({ "policies": [policy_name], "user": user }).to_string()),
)
.await?;
Ok(())
}
async fn create_user(env: &RustFSTestEnvironment, user: &str, secret: &str) -> TestResult {
admin_ok(
env,
http::Method::PUT,
&format!("/rustfs/admin/v3/add-user?accessKey={user}"),
Some(serde_json::json!({ "secretKey": secret, "status": "enabled" }).to_string()),
)
.await?;
Ok(())
}
async fn assert_access_denied(client: &Client, context: &str) -> TestResult {
async fn assert_chaining_denied(client: &Client, credential_kind: &str) -> TestResult {
let error = client
.assume_role()
.role_arn("arn:aws:iam::123456789012:role/test")
.role_session_name("sts-query-compat-e2e")
.send()
.await
.expect_err("AssumeRole must be denied");
.expect_err("credential chaining must be denied");
let service_error = error
.as_service_error()
.ok_or_else(|| format!("{context} should deserialize as an STS service error: {error:?}"))?;
.ok_or_else(|| format!("{credential_kind} denial should deserialize as an STS service error: {error:?}"))?;
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(403));
assert_eq!(service_error.code(), Some("AccessDenied"));
assert_eq!(service_error.message(), Some("Access Denied"));
assert!(
error.request_id().is_some_and(|request_id| !request_id.is_empty()),
"{context} should include a request ID"
"{credential_kind} denial should include a request ID"
);
Ok(())
}
async fn handle_opa_request(
request: Request<Incoming>,
requests: mpsc::UnboundedSender<Value>,
validation_started: mpsc::UnboundedSender<()>,
validation_mode: OpaValidationMode,
expected_authorization: Option<String>,
) -> Result<Response<Full<Bytes>>, Infallible> {
if let Some(expected_authorization) = expected_authorization
&& request.headers().get(AUTHORIZATION).and_then(|value| value.to_str().ok()) != Some(expected_authorization.as_str())
{
return Ok(Response::builder()
.status(401)
.body(Full::new(Bytes::new()))
.expect("static OPA unauthorized response must be valid"));
}
let body = match request.into_body().collect().await {
Ok(body) => body.to_bytes(),
Err(error) => {
return Ok(Response::builder()
.status(400)
.body(Full::new(Bytes::from(error.to_string())))
.expect("static OPA error response must be valid"));
}
};
let payload = if body.is_empty() {
None
} else {
match serde_json::from_slice::<Value>(&body) {
Ok(payload) => Some(payload),
Err(error) => {
return Ok(Response::builder()
.status(400)
.body(Full::new(Bytes::from(error.to_string())))
.expect("static OPA error response must be valid"));
}
}
};
if payload.is_none() {
let _ = validation_started.send(());
if let OpaValidationMode::DelayedUnavailable(release) = validation_mode {
release.notified().await;
return Ok(Response::builder()
.status(503)
.body(Full::new(Bytes::new()))
.expect("static OPA unavailable response must be valid"));
}
}
let allow = match payload.as_ref().and_then(|value| value.pointer("/input/identity/account")) {
Some(Value::String(account)) if account == "opaallow" => payload
.as_ref()
.and_then(|value| value.pointer("/input/context/deny_only"))
.and_then(Value::as_bool)
.unwrap_or(false),
Some(Value::String(account)) if account == "opadeny" => false,
None => true,
_ => false,
};
if let Some(payload) = payload {
let _ = requests.send(payload);
}
let body =
serde_json::to_vec(&serde_json::json!({ "result": { "allow": allow } })).expect("static OPA response must serialize");
Ok(Response::builder()
.header(CONTENT_TYPE, "application/json")
.body(Full::new(Bytes::from(body)))
.expect("static OPA response must be valid"))
}
#[derive(Clone)]
enum OpaValidationMode {
Ready,
DelayedUnavailable(Arc<Notify>),
}
struct OpaMock {
url: String,
requests: mpsc::UnboundedReceiver<Value>,
validation_started: mpsc::UnboundedReceiver<()>,
validation_release: Option<Arc<Notify>>,
task: JoinHandle<()>,
}
impl OpaMock {
async fn start() -> Result<Self, BoxError> {
Self::start_with_mode(OpaValidationMode::Ready, Some(OPA_AUTH_TOKEN)).await
}
async fn start_delayed_unavailable() -> Result<Self, BoxError> {
let release = Arc::new(Notify::new());
Self::start_with_mode(OpaValidationMode::DelayedUnavailable(release), None).await
}
async fn start_with_mode(validation_mode: OpaValidationMode, auth_token: Option<&str>) -> Result<Self, BoxError> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let url = format!("http://{}/v1/data/rustfs/authz/allow", listener.local_addr()?);
let (requests_tx, requests) = mpsc::unbounded_channel();
let (validation_started_tx, validation_started) = mpsc::unbounded_channel();
let expected_authorization = auth_token.map(|token| format!("Bearer {token}"));
let validation_release = match &validation_mode {
OpaValidationMode::Ready => None,
OpaValidationMode::DelayedUnavailable(release) => Some(Arc::clone(release)),
};
let task = tokio::spawn(async move {
let mut connections = JoinSet::new();
loop {
tokio::select! {
accepted = listener.accept() => {
let Ok((stream, _)) = accepted else { break };
let requests = requests_tx.clone();
let validation_started = validation_started_tx.clone();
let validation_mode = validation_mode.clone();
let expected_authorization = expected_authorization.clone();
connections.spawn(async move {
let handler = service_fn(move |request| {
handle_opa_request(
request,
requests.clone(),
validation_started.clone(),
validation_mode.clone(),
expected_authorization.clone(),
)
});
let _ = http1::Builder::new()
.serve_connection(TokioIo::new(stream), handler)
.await;
});
}
_ = connections.join_next(), if !connections.is_empty() => {}
}
}
});
Ok(Self {
url,
requests,
validation_started,
validation_release,
task,
})
}
async fn next_request(&mut self) -> Result<Value, BoxError> {
timeout(Duration::from_secs(5), self.requests.recv())
.await?
.ok_or_else(|| "OPA request channel closed".into())
}
async fn wait_for_validation(&mut self) -> TestResult {
timeout(Duration::from_secs(5), self.validation_started.recv())
.await?
.ok_or_else(|| "OPA validation channel closed".into())
}
fn release_validation(&self) {
if let Some(release) = &self.validation_release {
release.notify_one();
}
}
}
impl Drop for OpaMock {
fn drop(&mut self) {
self.task.abort();
}
}
#[tokio::test]
#[serial]
async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
@@ -359,7 +155,7 @@ async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
"signature rejection should include a request ID"
);
assert_access_denied(
assert_chaining_denied(
&sts_client(
&env.url,
temporary.access_key_id(),
@@ -371,187 +167,7 @@ async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
.await?;
let (service_access_key, service_secret_key) = create_root_service_account(&env).await?;
assert_access_denied(
&sts_client(&env.url, &service_access_key, &service_secret_key, None),
"service-account denial",
)
.await?;
let implicit_user = "stsimplicit";
let explicit_allow_user = "stsallow";
let explicit_deny_user = "stsdeny";
let policyless_user = "stspolicyless";
let secret = "stsAuthzSecret123";
create_user(&env, policyless_user, secret).await?;
assert_access_denied(&sts_client(&env.url, policyless_user, secret, None), "policyless user").await?;
create_user_with_policy(
&env,
implicit_user,
secret,
"sts-implicit-policy",
serde_json::json!([{
"Effect": "Allow",
"Action": ["s3:ListAllMyBuckets"],
"Resource": ["arn:aws:s3:::*"],
}]),
)
.await?;
create_user_with_policy(
&env,
explicit_allow_user,
secret,
"sts-allow-policy",
serde_json::json!([{
"Effect": "Allow",
"Action": ["sts:AssumeRole"],
"Resource": ["arn:aws:s3:::*"],
}]),
)
.await?;
create_user_with_policy(
&env,
explicit_deny_user,
secret,
"sts-deny-policy",
serde_json::json!([
{
"Effect": "Allow",
"Action": ["sts:AssumeRole"],
"Resource": ["arn:aws:s3:::*"],
},
{
"Effect": "Deny",
"Action": ["sts:AssumeRole"],
"Resource": ["arn:aws:s3:::*"],
}
]),
)
.await?;
for user in [implicit_user, explicit_allow_user] {
let output = sts_client(&env.url, user, secret, None)
.assume_role()
.role_arn("arn:aws:iam::123456789012:role/test")
.role_session_name("sts-authz-e2e")
.send()
.await
.map_err(|error| format!("{user} should be allowed to call AssumeRole: {error:?}"))?;
let credentials = output
.credentials()
.ok_or_else(|| format!("{user} AssumeRole response should contain credentials"))?;
assert!(!credentials.access_key_id().is_empty());
assert!(!credentials.secret_access_key().is_empty());
assert!(!credentials.session_token().is_empty());
}
assert_access_denied(&sts_client(&env.url, explicit_deny_user, secret, None), "explicit sts:AssumeRole Deny").await?;
env.stop_server();
Ok(())
}
#[tokio::test]
#[serial]
async fn test_sts_assume_role_opa_contract() -> TestResult {
init_logging();
let mut opa = OpaMock::start().await?;
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_with_env(
vec![],
&[
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
],
)
.await?;
let secret = "stsOpaSecret123";
create_user_with_policy(
&env,
"opaallow",
secret,
"sts-opa-local-deny-policy",
serde_json::json!([{
"Effect": "Deny",
"Action": ["sts:AssumeRole"],
"Resource": ["arn:aws:s3:::*"],
}]),
)
.await?;
create_user_with_policy(
&env,
"opadeny",
secret,
"sts-opa-local-allow-policy",
serde_json::json!([{
"Effect": "Allow",
"Action": ["sts:AssumeRole"],
"Resource": ["arn:aws:s3:::*"],
}]),
)
.await?;
sts_client(&env.url, "opaallow", secret, None)
.assume_role()
.role_arn("arn:aws:iam::123456789012:role/test")
.role_session_name("sts-opa-contract")
.send()
.await
.map_err(|error| format!("OPA allow should override the local explicit Deny: {error:?}"))?;
assert_access_denied(
&sts_client(&env.url, "opadeny", secret, None),
"OPA denial despite local sts:AssumeRole Allow",
)
.await?;
let mut accounts = BTreeSet::new();
for _ in 0..2 {
let request = opa.next_request().await?;
assert_eq!(request.pointer("/input/action").and_then(Value::as_str), Some("sts:AssumeRole"));
assert_eq!(request.pointer("/input/context/deny_only").and_then(Value::as_bool), Some(true));
let account = request
.pointer("/input/identity/account")
.and_then(Value::as_str)
.ok_or("OPA input should include identity.account")?;
accounts.insert(account.to_owned());
}
assert_eq!(accounts, BTreeSet::from(["opaallow".to_owned(), "opadeny".to_owned()]));
env.stop_server();
Ok(())
}
#[tokio::test]
#[serial]
async fn test_sts_assume_role_fails_closed_while_opa_is_unavailable() -> TestResult {
init_logging();
let mut opa = OpaMock::start_delayed_unavailable().await?;
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str())])
.await?;
opa.wait_for_validation().await?;
let user = "opaunavailable";
let secret = "stsOpaUnavailableSecret123";
create_user_with_policy(
&env,
user,
secret,
"sts-opa-unavailable-local-policy",
serde_json::json!([{
"Effect": "Allow",
"Action": ["s3:ListAllMyBuckets"],
"Resource": ["arn:aws:s3:::*"],
}]),
)
.await?;
assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA initialization").await?;
opa.release_validation();
tokio::time::sleep(Duration::from_millis(200)).await;
assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA validation failure").await?;
assert_chaining_denied(&sts_client(&env.url, &service_access_key, &service_secret_key, None), "service account").await?;
env.stop_server();
Ok(())
+9 -12
View File
@@ -172,9 +172,6 @@ pub mod bucket {
}
pub mod replication {
pub use crate::bucket::replication::replication_pool::{
DurableMrfBacklogSummary, DurableMrfBucketBacklog, durable_mrf_backlog_summary_snapshot,
};
pub use crate::bucket::replication::{
BucketReplicationResyncStatus, BucketStats, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool,
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REPLICATE_INCOMING_DELETE, ReplicateDecision,
@@ -422,15 +419,15 @@ pub mod rio {
pub mod rpc {
pub use crate::cluster::rpc::{
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
tonic_boot_epoch_response_headers, verify_rpc_signature, verify_tonic_boot_epoch_response,
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
};
}
@@ -46,11 +46,7 @@ use super::replication_target_boundary::{ReplicationTargetStore, replication_obj
use super::replication_versioning_boundary::ReplicationVersioningStore;
use super::runtime_boundary as runtime_sources;
use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str};
use std::collections::HashMap;
use std::collections::hash_map::Entry;
use std::sync::Arc;
use std::sync::LazyLock;
use std::sync::RwLock as StdRwLock;
use std::sync::atomic::AtomicI32;
use std::sync::atomic::Ordering;
use time::OffsetDateTime;
@@ -78,66 +74,6 @@ pub struct DurableMrfBacklog {
pub entries: Vec<MrfReplicateEntry>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct DurableMrfBucketBacklog {
pub bucket: String,
pub count: u64,
pub bytes: u64,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct DurableMrfBacklogSummary {
pub available: bool,
pub buckets: Vec<DurableMrfBucketBacklog>,
}
static DURABLE_MRF_BACKLOG_SUMMARY: LazyLock<StdRwLock<DurableMrfBacklogSummary>> =
LazyLock::new(|| StdRwLock::new(DurableMrfBacklogSummary::default()));
fn durable_mrf_backlog_summary_from_sizes<I>(entries: I) -> DurableMrfBacklogSummary
where
I: IntoIterator<Item = (String, i64)>,
{
let mut buckets = HashMap::<String, DurableMrfBucketBacklog>::new();
for (bucket_name, entry_size) in entries {
let Ok(size) = u64::try_from(entry_size) else {
return DurableMrfBacklogSummary::default();
};
let bucket = match buckets.entry(bucket_name) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
let bucket = entry.key().clone();
entry.insert(DurableMrfBucketBacklog {
bucket,
..Default::default()
})
}
};
bucket.count = bucket.count.saturating_add(1);
bucket.bytes = bucket.bytes.saturating_add(size);
}
DurableMrfBacklogSummary {
available: true,
buckets: buckets.into_values().collect(),
}
}
fn set_durable_mrf_backlog_summary(summary: DurableMrfBacklogSummary) {
match DURABLE_MRF_BACKLOG_SUMMARY.write() {
Ok(mut guard) => *guard = summary,
Err(poisoned) => *poisoned.into_inner() = summary,
}
}
pub fn durable_mrf_backlog_summary_snapshot() -> DurableMrfBacklogSummary {
match DURABLE_MRF_BACKLOG_SUMMARY.read() {
Ok(guard) => guard.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
}
}
fn durable_mrf_backlog_from_read(result: Result<Vec<u8>, EcstoreError>) -> DurableMrfBacklog {
match result {
Ok(data) => match decode_mrf_file(&data) {
@@ -297,7 +233,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
let active_counter = self.active_lrg_workers.clone();
let storage = self.storage.clone();
let stats = self.stats.clone();
let handle = tokio::spawn(async move {
let mut rx = rx;
@@ -306,18 +241,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
match operation {
ReplicationOperation::Object(obj_info) => {
let bucket = obj_info.bucket.clone();
let size = obj_info.size;
let delete_marker = obj_info.delete_marker;
let op_type = obj_info.op_type;
replicate_object(*obj_info, storage.clone()).await;
stats.dec_q(&bucket, size, delete_marker, op_type);
}
ReplicationOperation::Delete(del_info) => {
let bucket = del_info.bucket.clone();
let op_type = del_info.op_type;
replicate_delete(*del_info, storage.clone()).await;
stats.dec_q(&bucket, 0, true, op_type);
}
}
@@ -383,22 +310,23 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
match operation {
ReplicationOperation::Object(obj_info) => {
let bucket = obj_info.bucket.clone();
let size = obj_info.size;
let delete_marker = obj_info.delete_marker;
let op_type = obj_info.op_type;
// Perform actual replication (placeholder)
replicate_object(*obj_info, storage.clone()).await;
stats
.inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
.await;
stats.dec_q(&bucket, size, delete_marker, op_type);
// Perform actual replication (placeholder)
replicate_object(obj_info.as_ref().clone(), storage.clone()).await;
stats
.dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
.await;
}
ReplicationOperation::Delete(del_info) => {
let bucket = del_info.bucket.clone();
let op_type = del_info.op_type;
stats.inc_q(&del_info.bucket, 0, true, del_info.op_type).await;
// Perform actual delete replication (placeholder)
replicate_delete(*del_info, storage.clone()).await;
replicate_delete(del_info.as_ref().clone(), storage.clone()).await;
stats.dec_q(&bucket, 0, true, op_type);
stats.dec_q(&del_info.bucket, 0, true, del_info.op_type).await;
}
}
@@ -451,18 +379,16 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
active_counter.fetch_add(1, Ordering::SeqCst);
match operation {
ReplicationOperation::Object(obj_info) => {
let bucket = obj_info.bucket.clone();
let size = obj_info.size;
let delete_marker = obj_info.delete_marker;
let op_type = obj_info.op_type;
replicate_object(*obj_info, storage.clone()).await;
stats.dec_q(&bucket, size, delete_marker, op_type);
stats
.inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
.await;
replicate_object(obj_info.as_ref().clone(), storage.clone()).await;
stats
.dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
.await;
}
ReplicationOperation::Delete(del_info) => {
let bucket = del_info.bucket.clone();
let op_type = del_info.op_type;
replicate_delete(*del_info, storage.clone()).await;
stats.dec_q(&bucket, 0, true, op_type);
}
}
active_counter.fetch_sub(1, Ordering::SeqCst);
@@ -603,13 +529,9 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
if !lrg_workers.is_empty() {
let index = (hash as usize) % lrg_workers.len();
if let Some(worker) = lrg_workers.get(index) {
self.stats.inc_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
if worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_ok() {
return ReplicationQueueAdmission::Queued;
}
self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
if let Some(worker) = lrg_workers.get(index)
&& worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err()
{
// Try to add more workers if possible
let max_l_workers = *self.max_l_workers.read().await;
let existing = lrg_workers.len();
@@ -617,13 +539,17 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
drop(lrg_workers);
// Queue to MRF if worker is busy.
let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "large_object").await;
let admission =
queue_mrf_save_admission(&self.mrf_save_tx, ri.to_mrf_entry(), &ri.bucket, &ri.name, "large_object")
.await;
if let Some(resize) = resize {
self.resize_lrg_workers(resize.new_count, resize.existing_count).await;
}
return admission;
}
return ReplicationQueueAdmission::Queued;
}
return ReplicationQueueAdmission::Missed;
}
@@ -636,14 +562,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
return ReplicationQueueAdmission::Missed;
};
self.stats.inc_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
if channel.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_ok() {
return ReplicationQueueAdmission::Queued;
}
self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
// Queue to MRF if all workers are busy.
let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "object").await;
let admission = queue_mrf_save_admission(&self.mrf_save_tx, ri.to_mrf_entry(), &ri.bucket, &ri.name, "object").await;
// Try to scale up workers based on priority
self.apply_queue_backpressure("object", true, "Replication queue is backpressured")
@@ -662,13 +586,18 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
return ReplicationQueueAdmission::Missed;
};
self.stats.inc_q(&doi.bucket, 0, true, doi.op_type);
if channel.try_send(ReplicationOperation::Delete(Box::new(doi.clone()))).is_ok() {
return ReplicationQueueAdmission::Queued;
}
self.stats.dec_q(&doi.bucket, 0, true, doi.op_type);
let admission = self.queue_mrf_save_admission(doi.to_mrf_entry(), "delete").await;
let admission = queue_mrf_save_admission(
&self.mrf_save_tx,
doi.to_mrf_entry(),
&doi.bucket,
&doi.delete_object.object_name,
"delete",
)
.await;
self.apply_queue_backpressure("delete", false, "Replication delete queue is backpressured")
.await;
@@ -678,18 +607,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
/// Queues an MRF save operation
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
}
async fn queue_mrf_save_admission(&self, entry: MrfReplicateEntry, queue_type: &'static str) -> ReplicationQueueAdmission {
let bucket = entry.bucket.clone();
let size = entry.size;
let is_delete = matches!(entry.op, MrfOpKind::Delete);
let admission = queue_mrf_save_entry(&self.mrf_save_tx, entry, queue_type).await;
if admission == ReplicationQueueAdmission::Queued {
self.stats.inc_q(&bucket, size, is_delete, ReplicationType::Heal);
}
admission
let _ = queue_mrf_save_admission(&self.mrf_save_tx, entry, "", "", "mrf_worker").await;
}
/// Starts the MRF processor — one-shot at startup.
@@ -704,13 +622,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
let handle = tokio::spawn(async move {
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
Ok(d) => d,
Err(EcstoreError::ConfigNotFound) => {
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
available: true,
buckets: Vec::new(),
});
return;
}
Err(EcstoreError::ConfigNotFound) => return, // no file yet — normal on first start
Err(e) => {
warn!(
component = LOG_COMPONENT_ECSTORE,
@@ -741,9 +653,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
return;
}
};
set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
entries.iter().map(|entry| (entry.bucket.clone(), entry.size)),
));
let total = entries.len();
let mut queued_count = 0usize;
@@ -856,11 +765,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
error = %e,
"Failed to clear MRF recovery file after replay — entries may be replayed again on next restart"
);
} else {
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
available: true,
buckets: Vec::new(),
});
}
if queued_count > 0 {
@@ -889,7 +793,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
return;
};
let storage = self.storage.clone();
let stats = self.stats.clone();
let handle = tokio::spawn(async move {
// The on-disk MRF file is a restart-recovery backstop: entries are
@@ -915,7 +818,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
entry = rx.recv() => match entry {
Some(e) => {
if pending.len() >= MRF_PENDING_CAP {
dec_mrf_entries(stats.as_ref(), std::slice::from_ref(&e));
if !capped {
capped = true;
warn!(
@@ -934,31 +836,20 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
// set, not the absolute length, so a large backlog is
// not rewritten on every single add).
if pending.len() - flushed_len >= 1000 && flush_mrf_to_disk(&pending, &storage).await {
set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
pending.iter().map(|entry| (entry.bucket.clone(), entry.size)),
));
dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
flushed_len = pending.len();
dirty = false;
}
}
None => {
// Channel closed (pool shutting down) — final flush.
if dirty && flush_mrf_to_disk(&pending, &storage).await {
set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
pending.iter().map(|entry| (entry.bucket.clone(), entry.size)),
));
dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
if dirty {
flush_mrf_to_disk(&pending, &storage).await;
}
break;
}
},
_ = interval.tick() => {
if dirty && flush_mrf_to_disk(&pending, &storage).await {
set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
pending.iter().map(|entry| (entry.bucket.clone(), entry.size)),
));
dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
flushed_len = pending.len();
dirty = false;
}
@@ -981,22 +872,24 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
match operation {
ReplicationOperation::Object(obj_info) => {
let bucket = obj_info.bucket.clone();
let size = obj_info.size;
let delete_marker = obj_info.delete_marker;
let op_type = obj_info.op_type;
// Perform actual replication (placeholder)
replicate_object(*obj_info, self.storage.clone()).await;
stats
.inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
.await;
stats.dec_q(&bucket, size, delete_marker, op_type);
// Perform actual replication (placeholder)
replicate_object(obj_info.as_ref().clone(), self.storage.clone()).await;
stats
.dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
.await;
}
ReplicationOperation::Delete(del_info) => {
let bucket = del_info.bucket.clone();
let op_type = del_info.op_type;
// Perform actual delete replication (placeholder)
replicate_delete(*del_info, self.storage.clone()).await;
stats.inc_q(&del_info.bucket, 0, true, del_info.op_type).await;
stats.dec_q(&bucket, 0, true, op_type);
// Perform actual delete replication (placeholder)
replicate_delete(del_info.as_ref().clone(), self.storage.clone()).await;
stats.dec_q(&del_info.bucket, 0, true, del_info.op_type).await;
}
}
@@ -1005,30 +898,16 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
}
/// Worker function for handling large object replication operations
async fn add_large_worker(
&self,
mut rx: Receiver<ReplicationOperation>,
active_counter: Arc<AtomicI32>,
stats: Arc<ReplicationStats>,
storage: Arc<S>,
) {
async fn add_large_worker(&self, mut rx: Receiver<ReplicationOperation>, active_counter: Arc<AtomicI32>, storage: Arc<S>) {
while let Some(operation) = rx.recv().await {
active_counter.fetch_add(1, Ordering::SeqCst);
match operation {
ReplicationOperation::Object(obj_info) => {
let bucket = obj_info.bucket.clone();
let size = obj_info.size;
let delete_marker = obj_info.delete_marker;
let op_type = obj_info.op_type;
replicate_object(*obj_info, storage.clone()).await;
stats.dec_q(&bucket, size, delete_marker, op_type);
}
ReplicationOperation::Delete(del_info) => {
let bucket = del_info.bucket.clone();
let op_type = del_info.op_type;
replicate_delete(*del_info, storage.clone()).await;
stats.dec_q(&bucket, 0, true, op_type);
}
}
@@ -1048,19 +927,18 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
match operation {
ReplicationOperation::Object(obj_info) => {
let bucket = obj_info.bucket.clone();
let size = obj_info.size;
let delete_marker = obj_info.delete_marker;
let op_type = obj_info.op_type;
stats
.inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
.await;
replicate_object(obj_info.as_ref().clone(), self.storage.clone()).await;
stats.dec_q(&bucket, size, delete_marker, op_type);
stats
.dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
.await;
}
ReplicationOperation::Delete(del_info) => {
let bucket = del_info.bucket.clone();
let op_type = del_info.op_type;
replicate_delete(*del_info, self.storage.clone()).await;
stats.dec_q(&bucket, 0, true, op_type);
}
}
@@ -1395,34 +1273,29 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
}
}
async fn queue_mrf_save_entry(
async fn queue_mrf_save_admission(
tx: &Sender<MrfReplicateEntry>,
entry: MrfReplicateEntry,
bucket: &str,
object: &str,
queue_type: &'static str,
) -> ReplicationQueueAdmission {
let Err(error) = tx.send(entry).await else {
if tx.send(entry).await.is_ok() {
return ReplicationQueueAdmission::Queued;
};
let entry = error.0;
}
warn!(
event = EVENT_REPLICATION_MRF_QUEUE_UNAVAILABLE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket = %entry.bucket,
object = %entry.object,
bucket = %bucket,
object = %object,
queue_type = queue_type,
"MRF save channel unavailable — replication failure entry could not be persisted for retry"
);
ReplicationQueueAdmission::Missed
}
fn dec_mrf_entries(stats: &ReplicationStats, entries: &[MrfReplicateEntry]) {
for entry in entries {
stats.dec_q(&entry.bucket, entry.size, matches!(entry.op, MrfOpKind::Delete), ReplicationType::Heal);
}
}
/// Encodes `entries` and overwrites the MRF persistence file.
/// Returns `true` on success; on failure logs the error and returns `false`.
/// Callers must NOT clear their in-memory buffer on `false` so the next tick
@@ -2143,147 +2016,6 @@ mod tests {
})
}
async fn current_queue(pool: &ReplicationPool<LoadResyncNodeStore>, bucket: &str) -> (i64, i64) {
let stats = pool.stats.get_latest_replication_stats(bucket).await;
(stats.replication_stats.q_stat.curr.count, stats.replication_stats.q_stat.curr.bytes)
}
async fn wait_for_current_queue(pool: &ReplicationPool<LoadResyncNodeStore>, bucket: &str, expected: (i64, i64)) {
tokio::time::timeout(Duration::from_secs(10), async {
loop {
if current_queue(pool, bucket).await == expected {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("replication queue should reach the expected state");
}
#[tokio::test]
async fn regular_worker_admission_counts_channel_backlog_before_receive() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
let (tx, _rx) = mpsc::channel(1);
pool.workers.write().await.push(tx);
let admission = pool
.queue_replica_task(ReplicateObjectInfo {
bucket: "admission-bucket".to_string(),
name: "object".to_string(),
size: 4096,
op_type: ReplicationType::Object,
..Default::default()
})
.await;
assert_eq!(admission, ReplicationQueueAdmission::Queued);
assert_eq!(current_queue(&pool, "admission-bucket").await, (1, 4096));
}
#[tokio::test]
async fn large_worker_admission_counts_channel_backlog_before_receive() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
let (tx, _rx) = mpsc::channel(1);
pool.lrg_workers.write().await.push(tx);
let size = 128 * 1024 * 1024;
let admission = pool
.queue_replica_task(ReplicateObjectInfo {
bucket: "large-admission-bucket".to_string(),
name: "large-object".to_string(),
size,
op_type: ReplicationType::Object,
..Default::default()
})
.await;
assert_eq!(admission, ReplicationQueueAdmission::Queued);
assert_eq!(current_queue(&pool, "large-admission-bucket").await, (1, size));
}
#[tokio::test]
async fn delete_admission_counts_channel_backlog_before_receive() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
let (tx, _rx) = mpsc::channel(1);
pool.workers.write().await.push(tx);
let admission = pool
.queue_replica_delete_task(DeletedObjectReplicationInfo {
bucket: "delete-admission-bucket".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "deleted-object".to_string(),
..Default::default()
},
op_type: ReplicationType::Delete,
..Default::default()
})
.await;
assert_eq!(admission, ReplicationQueueAdmission::Queued);
assert_eq!(current_queue(&pool, "delete-admission-bucket").await, (1, 0));
}
#[tokio::test]
async fn regular_worker_drains_current_backlog_after_processing() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
pool.resize_workers(1, 0).await;
let admission = pool
.queue_replica_task(ReplicateObjectInfo {
bucket: "regular-drain-bucket".to_string(),
name: "object".to_string(),
size: 4096,
op_type: ReplicationType::Object,
..Default::default()
})
.await;
assert_eq!(admission, ReplicationQueueAdmission::Queued);
wait_for_current_queue(&pool, "regular-drain-bucket", (0, 0)).await;
}
#[tokio::test]
async fn large_worker_drains_current_backlog_after_processing() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
pool.resize_lrg_workers(1, 0).await;
let size = 128 * 1024 * 1024;
let admission = pool
.queue_replica_task(ReplicateObjectInfo {
bucket: "large-drain-bucket".to_string(),
name: "large-object".to_string(),
size,
op_type: ReplicationType::Object,
..Default::default()
})
.await;
assert_eq!(admission, ReplicationQueueAdmission::Queued);
wait_for_current_queue(&pool, "large-drain-bucket", (0, 0)).await;
}
#[tokio::test]
async fn regular_delete_worker_drains_current_backlog_after_processing() {
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
pool.resize_workers(1, 0).await;
let admission = pool
.queue_replica_delete_task(DeletedObjectReplicationInfo {
bucket: "delete-drain-bucket".to_string(),
delete_object: ReplicationDeletedObject {
object_name: "deleted-object".to_string(),
..Default::default()
},
op_type: ReplicationType::Delete,
..Default::default()
})
.await;
assert_eq!(admission, ReplicationQueueAdmission::Queued);
wait_for_current_queue(&pool, "delete-drain-bucket", (0, 0)).await;
}
fn load_resync_test_metadata() -> Vec<u8> {
let mut status = BucketReplicationResyncStatus::new();
status.targets_map.insert(
@@ -2569,37 +2301,6 @@ mod tests {
assert_eq!(admission, ReplicationQueueAdmission::Missed);
}
#[tokio::test]
async fn queue_replica_task_counts_mrf_pending_backlog_when_worker_queue_is_full() {
let shared = empty_resync_shared_state();
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", shared))).await;
let (tx, _rx) = mpsc::channel(1);
tx.try_send(ReplicationOperation::Object(Box::new(ReplicateObjectInfo {
bucket: "runtime-backlog".to_string(),
name: "already-buffered".to_string(),
size: 1,
op_type: ReplicationType::Object,
..Default::default()
})))
.expect("test setup should fill the worker queue");
pool.workers.write().await.push(tx);
let admission = pool
.queue_replica_task(ReplicateObjectInfo {
bucket: "runtime-backlog".to_string(),
name: "fallback-object".to_string(),
size: 2048,
op_type: ReplicationType::Object,
..Default::default()
})
.await;
assert_eq!(admission, ReplicationQueueAdmission::Queued);
let queued = pool.stats.get_latest_replication_stats("runtime-backlog").await;
assert_eq!(queued.replication_stats.q_stat.curr.count, 1);
assert_eq!(queued.replication_stats.q_stat.curr.bytes, 2048);
}
#[test]
fn replicate_object_info_from_object_info_preserves_ssec_checksum() {
let checksum = bytes::Bytes::from_static(b"ssec-checksum");
@@ -2641,7 +2342,7 @@ mod tests {
tx.try_send(first).expect("first MRF entry should fill the test channel");
let admission = queue_mrf_save_entry(&tx, second, "test");
let admission = queue_mrf_save_admission(&tx, second, "bucket", "second", "test");
tokio::pin!(admission);
assert!(
@@ -2986,30 +2687,6 @@ mod tests {
assert!(missing_file.entries.is_empty());
}
#[test]
fn durable_mrf_summary_aggregates_entries_by_bucket_for_obs() {
let summary =
durable_mrf_backlog_summary_from_sizes([("b1".to_string(), 1024), ("b1".to_string(), 512), ("b2".to_string(), 0)]);
assert!(summary.available);
let buckets = summary
.buckets
.into_iter()
.map(|bucket| (bucket.bucket.clone(), bucket))
.collect::<HashMap<_, _>>();
assert_eq!(buckets["b1"].count, 2);
assert_eq!(buckets["b1"].bytes, 1536);
assert_eq!(buckets["b2"].count, 1);
assert_eq!(buckets["b2"].bytes, 0);
}
#[test]
fn durable_mrf_summary_marks_invalid_sizes_unavailable() {
let invalid = durable_mrf_backlog_summary_from_sizes([("bucket".to_string(), -1)]);
assert!(!invalid.available);
assert!(invalid.buckets.is_empty());
}
#[test]
fn durable_mrf_snapshot_marks_corrupt_or_invalid_data_unavailable() {
let corrupt = durable_mrf_backlog_from_read(Ok(vec![0, 1, 2]));
@@ -23,8 +23,8 @@ use super::replication_stats_boundary::{
};
use super::runtime_boundary as runtime_sources;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::{Arc, Mutex as StdMutex};
use std::time::{Duration, SystemTime};
use tokio::sync::{Mutex, RwLock};
use tokio::time::interval;
@@ -143,7 +143,7 @@ pub struct ReplicationStats {
// Active worker statistics
pub workers: Arc<Mutex<ActiveWorkerStat>>,
// Queue statistics cache
pub q_cache: Arc<StdMutex<QueueCache>>,
pub q_cache: Arc<Mutex<QueueCache>>,
// Proxy statistics cache
pub p_cache: Arc<Mutex<ProxyStatsCache>>,
// MRF backlog statistics (simplified)
@@ -158,7 +158,7 @@ impl ReplicationStats {
Self {
sr_stats: Arc::new(SRStats::new()),
workers: Arc::new(Mutex::new(ActiveWorkerStat::new())),
q_cache: Arc::new(StdMutex::new(QueueCache::new())),
q_cache: Arc::new(Mutex::new(QueueCache::new())),
p_cache: Arc::new(Mutex::new(ProxyStatsCache::new())),
mrf_stats: HashMap::new(),
cache: Arc::new(RwLock::new(HashMap::new())),
@@ -198,9 +198,8 @@ impl ReplicationStats {
let mut interval = interval(Duration::from_secs(2));
loop {
interval.tick().await;
if let Ok(mut cache) = q_cache_clone.lock() {
cache.update();
}
let mut cache = q_cache_clone.lock().await;
cache.update();
}
});
}
@@ -392,13 +391,12 @@ impl ReplicationStats {
drop(cache);
{
if let Ok(q_cache) = self.q_cache.lock() {
for (bucket, queue_stats) in &q_cache.bucket_stats {
let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
bucket_stats.q_stat = queue_stats.snapshot();
bucket_stats.mark_node_local_provider_available();
bucket_stats.queue_scope = ReplicationMetricScope::NodeLocal;
}
let q_cache = self.q_cache.lock().await;
for (bucket, queue_stats) in &q_cache.bucket_stats {
let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
bucket_stats.q_stat = queue_stats.snapshot();
bucket_stats.mark_node_local_provider_available();
bucket_stats.queue_scope = ReplicationMetricScope::NodeLocal;
}
}
@@ -431,11 +429,8 @@ impl ReplicationStats {
let boot_time = SystemTime::UNIX_EPOCH; // simplified implementation
let uptime = SystemTime::now().duration_since(boot_time).unwrap_or_default().as_secs() as i64;
let queued = self
.q_cache
.lock()
.map(|q_cache| q_cache.get_site_stats())
.unwrap_or_default();
let q_cache = self.q_cache.lock().await;
let queued = q_cache.get_site_stats();
let p_cache = self.p_cache.lock().await;
let proxied = p_cache.get_site_stats();
@@ -638,9 +633,8 @@ impl ReplicationStats {
drop(cache);
{
if let Ok(q_cache) = self.q_cache.lock()
&& let Some(queue_stats) = q_cache.bucket_stats.get(bucket)
{
let q_cache = self.q_cache.lock().await;
if let Some(queue_stats) = q_cache.bucket_stats.get(bucket) {
replication_stats.q_stat = queue_stats.snapshot();
}
}
@@ -671,17 +665,31 @@ impl ReplicationStats {
}
/// Increase queue statistics
pub fn inc_q(&self, bucket: &str, size: i64, _is_delete_repl: bool, _op_type: ReplicationType) {
if let Ok(mut q_cache) = self.q_cache.lock() {
q_cache.inc(bucket, size);
}
pub async fn inc_q(&self, bucket: &str, size: i64, _is_delete_repl: bool, _op_type: ReplicationType) {
let mut q_cache = self.q_cache.lock().await;
let stats = q_cache
.bucket_stats
.entry(bucket.to_string())
.or_insert_with(InQueueMetric::default);
stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
q_cache.sr_queue_stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
q_cache.sr_queue_stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
}
/// Decrease queue statistics
pub fn dec_q(&self, bucket: &str, size: i64, _is_del_marker: bool, _op_type: ReplicationType) {
if let Ok(mut q_cache) = self.q_cache.lock() {
q_cache.dec(bucket, size);
}
pub async fn dec_q(&self, bucket: &str, size: i64, _is_del_marker: bool, _op_type: ReplicationType) {
let mut q_cache = self.q_cache.lock().await;
let stats = q_cache
.bucket_stats
.entry(bucket.to_string())
.or_insert_with(InQueueMetric::default);
stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
q_cache.sr_queue_stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
q_cache.sr_queue_stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
}
/// Increase proxy metrics
@@ -793,14 +801,14 @@ mod tests {
async fn latest_stats_include_queue_until_drained() {
let stats = ReplicationStats::new();
stats.inc_q("queued-bucket", 4096, false, ReplicationType::Object);
stats.inc_q("queued-bucket", 4096, false, ReplicationType::Object).await;
let queued = stats.get_latest_replication_stats("queued-bucket").await;
assert!(queued.replication_stats.provider_available);
assert_eq!(queued.replication_stats.q_stat.curr.count, 1);
assert_eq!(queued.replication_stats.q_stat.curr.bytes, 4096);
assert_eq!(queued.replication_stats.queue_scope, ReplicationMetricScope::NodeLocal);
stats.dec_q("queued-bucket", 4096, false, ReplicationType::Object);
stats.dec_q("queued-bucket", 4096, false, ReplicationType::Object).await;
let drained = stats.get_latest_replication_stats("queued-bucket").await;
assert_eq!(drained.replication_stats.q_stat.curr.count, 0);
assert_eq!(drained.replication_stats.q_stat.curr.bytes, 0);
@@ -903,7 +911,7 @@ mod tests {
for _ in 0..32 {
let stats = Arc::clone(&stats);
tasks.push(tokio::spawn(async move {
stats.inc_q("concurrent-bucket", 7, false, ReplicationType::Object);
stats.inc_q("concurrent-bucket", 7, false, ReplicationType::Object).await;
}));
}
for task in tasks {
+1 -1
View File
@@ -44,7 +44,7 @@ pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
pub use internode_data_transport::build_internode_data_transport_from_env;
pub(crate) use peer_rest_client::TierConfigReloadOutcome;
pub use peer_rest_client::{
KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
};
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
@@ -45,9 +45,9 @@ use rustfs_protos::proto_gen::node_service::{
HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ScannerActivityRequest,
ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest,
StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
tier_mutation_control_service_client::TierMutationControlServiceClient,
};
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
@@ -71,12 +71,6 @@ use uuid::Uuid;
pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
/// Dynamic config subsystem for the cluster-persisted KMS configuration.
///
/// KMS configuration lives in its own cluster object rather than in the server
/// config document, so it is not a `ServerConfig` subsystem; it only shares the
/// reload signal transport.
pub const KMS_SIGNAL_SUBSYSTEM: &str = "kms";
const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
const HEAL_CONTROL_FINGERPRINT_MAX_SIZE: usize = 256;
const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
@@ -105,13 +99,8 @@ fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<
}
fn validate_signal_service_protocol(sig: u64, sub_sys: &str, protocol_version: u32) -> Result<()> {
// The version stays pinned to DYNAMIC_CONFIG_PROTOCOL_VERSION rather than
// being bumped per subsystem: the comparison is shared, so raising it would
// retire peers that already converge scanner and heal config correctly.
// Subsystems added after a peer was built are rejected by that peer's own
// subsystem allow-list, which surfaces as an explicit failed signal.
if sig == SERVICE_SIGNAL_RELOAD_DYNAMIC
&& matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS | KMS_SIGNAL_SUBSYSTEM)
&& matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS)
&& protocol_version < rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION
{
return Err(Error::other(format!("peer does not support dynamic {sub_sys} config convergence")));
@@ -1511,22 +1500,6 @@ impl PeerRestClient {
}
pub async fn signal_service(&self, sig: u64, sub_sys: &str, dry_run: bool, _exec_at: SystemTime) -> Result<()> {
self.signal_service_checked(sig, sub_sys, dry_run).await.map(|_| ())
}
/// Report the KMS configuration fingerprint the peer is currently running.
///
/// Sent as a dry-run reload signal so the peer answers without swapping its
/// own configuration. `None` means the peer has no KMS configuration. The
/// fingerprint is advisory and feeds cluster status reporting only, so the
/// response is not proof-signed.
pub async fn kms_config_fingerprint(&self) -> Result<Option<String>> {
self.signal_service_checked(SERVICE_SIGNAL_RELOAD_DYNAMIC, KMS_SIGNAL_SUBSYSTEM, true)
.await
.map(|response| response.config_fingerprint)
}
async fn signal_service_checked(&self, sig: u64, sub_sys: &str, dry_run: bool) -> Result<SignalServiceResponse> {
self.finalize_result(
async {
let mut client = self.get_client().await?;
@@ -1547,7 +1520,7 @@ impl PeerRestClient {
return Err(Error::other(""));
}
validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
Ok(response)
Ok(())
}
.await,
)
@@ -2374,19 +2347,6 @@ mod tests {
.expect("full refresh compatibility is guarded by its scanner preflight");
}
#[test]
fn dynamic_kms_config_requires_versioned_peer_acknowledgement() {
let err = validate_signal_service_protocol(SERVICE_SIGNAL_RELOAD_DYNAMIC, KMS_SIGNAL_SUBSYSTEM, 0)
.expect_err("an unversioned peer must not claim KMS config convergence");
assert!(err.to_string().contains("does not support dynamic"));
validate_signal_service_protocol(
SERVICE_SIGNAL_RELOAD_DYNAMIC,
KMS_SIGNAL_SUBSYSTEM,
rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION,
)
.expect("a current peer should support dynamic KMS config");
}
#[test]
fn peer_rest_client_marks_network_like_errors() {
assert!(PeerRestClient::is_network_like_error(&Error::other("transport error")));
+2 -2
View File
@@ -5078,7 +5078,7 @@ impl LocalDisk {
Ok((buf, mtime))
}
#[hotpath::measure(impl_type = "LocalDisk")]
#[hotpath::measure]
async fn read_metadata_with_dmtime(&self, file_path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
check_path_length(file_path.as_ref().to_string_lossy().as_ref())?;
@@ -5121,7 +5121,7 @@ impl LocalDisk {
Ok((data, modtime))
}
#[hotpath::measure(impl_type = "LocalDisk")]
#[hotpath::measure]
async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
// TODO: timeout support
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
+2 -2
View File
@@ -103,7 +103,7 @@ where
/// or `out` is larger than one shard. On error `out`'s contents are
/// unspecified but never contain bytes that failed the hash check — the copy
/// happens only after verification.
#[hotpath::measure(impl_type = "BitrotReader")]
#[hotpath::measure]
pub async fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
let want = out.len();
self.begin_read(want)?;
@@ -303,7 +303,7 @@ where
/// Write a (hash+data) block. Returns the number of data bytes written.
/// Returns an error if called after a short write or if data exceeds shard_size.
#[hotpath::measure(label = "BitrotWriter::write", impl_type = "BitrotWriter")]
#[hotpath::measure(label = "BitrotWriter::write")]
pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if buf.is_empty() {
return Ok(0);
+2 -2
View File
@@ -691,7 +691,7 @@ impl<R> ParallelReader<R>
where
R: crate::erasure::coding::ShardSource,
{
#[hotpath::measure(impl_type = "ParallelReader")]
#[hotpath::measure]
pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
// On the reconstruction-verifying GET path, read every live shard reader
// in lockstep so all readers advance one block per stripe and stay
@@ -1505,7 +1505,7 @@ where
}
impl Erasure {
#[hotpath::measure(impl_type = "Erasure")]
#[hotpath::measure]
pub async fn decode<W, R>(
&self,
writer: &mut W,
+56 -534
View File
@@ -28,11 +28,8 @@ use std::vec;
use tokio::io::AsyncRead;
use tokio::runtime::RuntimeFlavor;
use tokio::sync::mpsc;
use tokio::task::{JoinError, JoinHandle};
use tracing::error;
/// Queue-capacity input for encoded blocks awaiting shard writers; it is not a
/// per-PUT or process-RSS memory limit.
const ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: &str = "RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES";
const ENV_RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS: &str = "RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS";
const ENV_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST: &str = "RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST";
@@ -90,33 +87,6 @@ fn use_bytesmut_ingest() -> bool {
rustfs_utils::get_env_bool(ENV_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST, DEFAULT_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST)
})
}
/// Keeps the encoder producer scoped to its parent future. Tokio detaches a
/// task when its `JoinHandle` is dropped, so the producer must be aborted when
/// an upload is cancelled before the encode pipeline finishes.
struct AbortOnDropTask<T>(JoinHandle<T>);
impl<T> AbortOnDropTask<T> {
fn new(task: JoinHandle<T>) -> Self {
Self(task)
}
async fn abort_and_wait(&mut self) {
self.0.abort();
let _ = (&mut self.0).await;
}
async fn join(&mut self) -> Result<T, JoinError> {
(&mut self.0).await
}
}
impl<T> Drop for AbortOnDropTask<T> {
fn drop(&mut self) {
self.0.abort();
}
}
/// Read up to `limit` bytes into `buf`'s uninitialized spare capacity, appending after its
/// current length, and distinguish a clean EOF from a short read.
///
@@ -163,65 +133,22 @@ fn queued_block_bytes(block: &[Bytes]) -> usize {
block.iter().map(Bytes::len).sum()
}
/// Owns an encoded queue entry's gauge contribution until its consumer takes it.
struct QueuedInflightBytes {
bytes: usize,
}
impl QueuedInflightBytes {
fn new(bytes: usize) -> Self {
rustfs_io_metrics::add_ec_encode_inflight_bytes(bytes);
Self { bytes }
async fn drain_queued_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Bytes>>) {
while let Some(block) = rx.recv().await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_block_bytes(&block));
}
fn settle(&mut self) {
let bytes = std::mem::take(&mut self.bytes);
if bytes != 0 {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(bytes);
}
}
}
impl Drop for QueuedInflightBytes {
fn drop(&mut self) {
self.settle();
}
}
/// Couples an encoded block with its queue gauge contribution. Keeping the
/// guard in the queue entry also covers Tokio sends that complete through a
/// permit after the receiver has closed.
struct InflightEntry<T> {
entry: T,
accounting: QueuedInflightBytes,
}
impl<T> InflightEntry<T> {
fn new(entry: T, bytes: usize) -> Self {
Self {
entry,
accounting: QueuedInflightBytes::new(bytes),
}
}
fn into_inner(mut self) -> T {
self.accounting.settle();
self.entry
}
}
async fn send_queued<T>(
sender: &mpsc::Sender<InflightEntry<T>>,
entry: T,
bytes: usize,
) -> Result<(), mpsc::error::SendError<InflightEntry<T>>> {
sender.send(InflightEntry::new(entry, bytes)).await
}
fn queued_batch_bytes(batch: &[Vec<Bytes>]) -> usize {
batch.iter().map(|block| queued_block_bytes(block)).sum()
}
async fn drain_queued_batched_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Vec<Bytes>>>) {
while let Some(batch) = rx.recv().await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
}
}
fn dominant_error_summary_label(summary: &WriteQuorumFailureSummary) -> &'static str {
summary.dominant_error_label
}
@@ -577,7 +504,7 @@ impl Erasure {
Ok((reader, total))
}
#[hotpath::measure(impl_type = "Erasure")]
#[hotpath::measure]
pub async fn encode<R>(
self: Arc<Self>,
reader: R,
@@ -613,14 +540,13 @@ impl Erasure {
));
}
// Bound queued encoded blocks by a queue budget; this does not bound
// reader, encoder, writer, allocator, or process-RSS memory.
// Bound queued encoded blocks by memory budget to avoid per-request spikes.
let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
let max_inflight_bytes = erasure_encode_max_inflight_bytes();
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Bytes>>>(inflight_blocks);
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(inflight_blocks);
let mut task = AbortOnDropTask::new(tokio::spawn(async move {
let task = tokio::spawn(async move {
let block_size = self.block_size;
let mut total = 0;
if use_bytesmut_ingest {
@@ -641,8 +567,10 @@ impl Erasure {
let res = self.clone().encode_block_bytes_mut(encode_buf, n).await?;
buf = BytesMut::with_capacity(ingest_capacity);
let queued_bytes = queued_block_bytes(&res);
rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = send_queued(&tx, res, queued_bytes).await {
if let Err(err) = tx.send(res).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
@@ -670,8 +598,10 @@ impl Erasure {
let (res, returned_buf) = self.clone().encode_block(encode_buf, n).await?;
buf = returned_buf;
let queued_bytes = queued_block_bytes(&res);
rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = send_queued(&tx, res, queued_bytes).await {
if let Err(err) = tx.send(res).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
@@ -696,7 +626,7 @@ impl Erasure {
}
Ok((reader, total))
}));
});
let mut writers = MultiWriter::new(writers, quorum);
@@ -708,10 +638,11 @@ impl Erasure {
break;
};
record_internal_stage_if_enabled("erasure_encode_recv_wait", recv_wait_stage_start);
let block = block.into_inner();
if block.is_empty() {
break;
}
let queued_bytes = queued_block_bytes(&block);
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
let write_stage_start = stage_timer_if_enabled();
if let Err(err) = writers.write(block).await {
write_err = Some(err);
@@ -721,8 +652,9 @@ impl Erasure {
}
if let Some(err) = write_err {
task.abort_and_wait().await;
drop(rx);
task.abort();
let _ = task.await;
drain_queued_inflight_bytes(&mut rx).await;
let shutdown_stage_start = stage_timer_if_enabled();
if let Err(shutdown_err) = writers.shutdown().await {
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
@@ -731,14 +663,14 @@ impl Erasure {
return Err(err);
}
let (reader, total) = task.join().await??;
let (reader, total) = task.await??;
let shutdown_stage_start = stage_timer_if_enabled();
writers.shutdown().await?;
record_internal_stage_if_enabled("erasure_encode_shutdown", shutdown_stage_start);
Ok((reader, total))
}
#[hotpath::measure(impl_type = "Erasure")]
#[hotpath::measure]
pub async fn encode_batched<R>(
self: Arc<Self>,
mut reader: R,
@@ -760,9 +692,9 @@ impl Erasure {
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
let batch_blocks = encode_batch_block_count().min(inflight_blocks);
let channel_capacity = inflight_blocks.div_ceil(batch_blocks).max(1);
let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Vec<Bytes>>>>(channel_capacity);
let (tx, mut rx) = mpsc::channel::<Vec<Vec<Bytes>>>(channel_capacity);
let mut task = AbortOnDropTask::new(tokio::spawn(async move {
let task = tokio::spawn(async move {
let block_size = self.block_size;
let mut total = 0;
let mut buf = vec![0u8; block_size];
@@ -781,8 +713,10 @@ impl Erasure {
pending_batch.push(res);
if pending_batch.len() >= batch_blocks {
rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = send_queued(&tx, pending_batch, pending_batch_bytes).await {
if let Err(err) = tx.send(pending_batch).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
@@ -808,15 +742,17 @@ impl Erasure {
}
if !pending_batch.is_empty() {
rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
let send_wait_stage_start = stage_timer_if_enabled();
if let Err(err) = send_queued(&tx, pending_batch, pending_batch_bytes).await {
if let Err(err) = tx.send(pending_batch).await {
rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
}
Ok((reader, total))
}));
});
let mut writers = MultiWriter::new(writers, quorum);
let mut write_err = None;
@@ -827,7 +763,7 @@ impl Erasure {
break;
};
record_internal_stage_if_enabled("erasure_encode_batched_recv_wait", recv_wait_stage_start);
let batch = batch.into_inner();
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
let write_stage_start = stage_timer_if_enabled();
for block in batch {
if let Err(err) = writers.write(block).await {
@@ -842,8 +778,9 @@ impl Erasure {
}
if let Some(err) = write_err {
task.abort_and_wait().await;
drop(rx);
task.abort();
let _ = task.await;
drain_queued_batched_inflight_bytes(&mut rx).await;
let shutdown_stage_start = stage_timer_if_enabled();
if let Err(shutdown_err) = writers.shutdown().await {
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
@@ -852,7 +789,7 @@ impl Erasure {
return Err(err);
}
let (reader, total) = task.join().await??;
let (reader, total) = task.await??;
let shutdown_stage_start = stage_timer_if_enabled();
writers.shutdown().await?;
record_internal_stage_if_enabled("erasure_encode_batched_shutdown", shutdown_stage_start);
@@ -861,7 +798,7 @@ impl Erasure {
/// Fast path for small inline objects: skip tokio::spawn + mpsc channel.
/// Reads all data, encodes directly, writes shards sequentially.
#[hotpath::measure(impl_type = "Erasure")]
#[hotpath::measure]
pub async fn encode_inline_small<R>(
self: Arc<Self>,
reader: R,
@@ -876,7 +813,7 @@ impl Erasure {
/// Fast path for single-block non-inline objects: avoids the producer/consumer
/// pipeline in `encode()` while keeping the same writer/quorum/shutdown semantics.
#[hotpath::measure(impl_type = "Erasure")]
#[hotpath::measure]
pub async fn encode_single_block_non_inline<R>(
self: Arc<Self>,
reader: R,
@@ -902,104 +839,7 @@ mod tests {
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use std::time::Duration;
use tokio::io::{AsyncWrite, AsyncWriteExt, ReadBuf};
use tokio::sync::oneshot;
struct PendingReader {
entered: Option<oneshot::Sender<()>>,
dropped: Option<oneshot::Sender<()>>,
}
impl PendingReader {
fn new() -> (Self, oneshot::Receiver<()>, oneshot::Receiver<()>) {
let (entered_tx, entered_rx) = oneshot::channel();
let (dropped_tx, dropped_rx) = oneshot::channel();
(
Self {
entered: Some(entered_tx),
dropped: Some(dropped_tx),
},
entered_rx,
dropped_rx,
)
}
}
impl AsyncRead for PendingReader {
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
if let Some(entered) = self.entered.take() {
let _ = entered.send(());
}
Poll::Pending
}
}
impl Drop for PendingReader {
fn drop(&mut self) {
if let Some(dropped) = self.dropped.take() {
let _ = dropped.send(());
}
}
}
struct BlocksThenPendingReader {
blocks_remaining: usize,
block: Vec<u8>,
blocked: Option<oneshot::Sender<()>>,
dropped: Option<oneshot::Sender<()>>,
final_block: Option<oneshot::Sender<()>>,
}
impl BlocksThenPendingReader {
fn new(
blocks_remaining: usize,
block_size: usize,
) -> (Self, oneshot::Receiver<()>, oneshot::Receiver<()>, oneshot::Receiver<()>) {
let (blocked_tx, blocked_rx) = oneshot::channel();
let (dropped_tx, dropped_rx) = oneshot::channel();
let (final_block_tx, final_block_rx) = oneshot::channel();
(
Self {
blocks_remaining,
block: vec![0x5a; block_size],
blocked: Some(blocked_tx),
dropped: Some(dropped_tx),
final_block: Some(final_block_tx),
},
blocked_rx,
dropped_rx,
final_block_rx,
)
}
}
impl AsyncRead for BlocksThenPendingReader {
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
if self.blocks_remaining == 0 {
if let Some(blocked) = self.blocked.take() {
let _ = blocked.send(());
}
return Poll::Pending;
}
if self.blocks_remaining == 1
&& let Some(final_block) = self.final_block.take()
{
let _ = final_block.send(());
}
self.blocks_remaining -= 1;
buf.put_slice(&self.block);
Poll::Ready(Ok(()))
}
}
impl Drop for BlocksThenPendingReader {
fn drop(&mut self) {
if let Some(dropped) = self.dropped.take() {
let _ = dropped.send(());
}
}
}
use tokio::io::{AsyncWrite, AsyncWriteExt};
fn erasure_with_zero_block_size() -> Erasure {
let mut erasure = Erasure::default();
@@ -1057,54 +897,6 @@ mod tests {
}
}
struct FailAfterReaderBlocksWriter {
reader_blocked: oneshot::Receiver<()>,
writes: Arc<std::sync::atomic::AtomicUsize>,
}
impl AsyncWrite for FailAfterReaderBlocksWriter {
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
match Pin::new(&mut self.reader_blocked).poll(cx) {
Poll::Pending => Poll::Pending,
Poll::Ready(_) => {
self.writes.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Poll::Ready(Err(std::io::Error::other("injected write failure after producer blocks")))
}
}
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
}
struct StallOnWriteWithSignal {
entered: Option<oneshot::Sender<()>>,
writes: Arc<std::sync::atomic::AtomicUsize>,
}
impl AsyncWrite for StallOnWriteWithSignal {
fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
self.writes.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if let Some(entered) = self.entered.take() {
let _ = entered.send(());
}
Poll::Pending
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
}
#[derive(Clone, Default)]
struct ShortWriteWriter;
@@ -1254,202 +1046,6 @@ mod tests {
BitrotWriterWrapper::new(CustomWriter::new_tokio_writer(writer), shard_size, HashAlgorithm::None)
}
#[derive(Clone, Copy)]
enum EncodePipeline {
Vec,
BytesMut,
Batched,
}
async fn aborting_encode_drops_blocked_producer(pipeline: EncodePipeline) {
const BLOCK_SIZE: usize = 16;
let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
let committed = Arc::new(Mutex::new(Vec::new()));
let mut writers = vec![Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE))];
let (reader, entered, dropped) = PendingReader::new();
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
let encode = match pipeline {
EncodePipeline::Vec => {
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await })
}
EncodePipeline::BytesMut => {
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await })
}
EncodePipeline::Batched => tokio::spawn(async move { erasure.encode_batched(reader, &mut writers, 1).await }),
};
tokio::time::timeout(Duration::from_secs(1), entered)
.await
.expect("producer should enter the blocked reader before cancellation")
.expect("blocked reader should signal entry");
encode.abort();
assert!(matches!(encode.await, Err(err) if err.is_cancelled()), "encode task should be cancelled");
tokio::time::timeout(Duration::from_secs(1), dropped)
.await
.expect("cancelling encode should drop the producer reader")
.expect("blocked reader should signal producer drop");
assert!(
committed.lock().expect("committed buffer should be lockable").is_empty(),
"cancelling before the first encoded block must not make data visible"
);
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
gauge_baseline,
"cancelling the encode pipeline must preserve the inflight queue gauge"
);
}
async fn writer_error_aborts_blocked_producer(pipeline: EncodePipeline) {
const BLOCK_SIZE: usize = 16;
let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
let blocks_before_pending = match pipeline {
EncodePipeline::Batched => encode_batch_block_count(),
EncodePipeline::Vec | EncodePipeline::BytesMut => 1,
};
let (reader, reader_blocked, reader_dropped, _final_block) =
BlocksThenPendingReader::new(blocks_before_pending, BLOCK_SIZE);
let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let mut writers = vec![Some(bitrot_writer(
FailAfterReaderBlocksWriter {
reader_blocked,
writes: writes.clone(),
},
BLOCK_SIZE,
))];
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
let result = match pipeline {
EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await,
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await,
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
};
let err = match result {
Ok(_) => panic!("writer quorum failure should fail the encode pipeline"),
Err(err) => err,
};
assert!(err.to_string().contains("Failed to write data"));
tokio::time::timeout(Duration::from_secs(1), reader_dropped)
.await
.expect("writer failure should abort the blocked producer")
.expect("blocked producer should signal reader drop");
assert_eq!(
writes.load(std::sync::atomic::Ordering::SeqCst),
1,
"writer failure must stop the pipeline before any additional shard write"
);
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
gauge_baseline,
"writer failure must settle all queued and pending encoded bytes"
);
}
async fn aborting_full_queue_settles_pending_send() {
const BLOCK_SIZE: usize = 16;
let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
let inflight_blocks = encode_channel_capacity(
erasure.shard_size().saturating_mul(erasure.total_shard_count()),
erasure_encode_max_inflight_bytes(),
);
let (reader, _reader_blocked, reader_dropped, final_block) =
BlocksThenPendingReader::new(inflight_blocks + 2, BLOCK_SIZE);
let (writer_entered_tx, writer_entered) = oneshot::channel();
let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let mut writers = vec![Some(bitrot_writer(
StallOnWriteWithSignal {
entered: Some(writer_entered_tx),
writes: writes.clone(),
},
BLOCK_SIZE,
))];
let erasure_for_task = erasure.clone();
let encode = tokio::spawn(async move { erasure_for_task.encode_with_ingest_mode(reader, &mut writers, 1, false).await });
tokio::time::timeout(Duration::from_secs(1), writer_entered)
.await
.expect("consumer should start the first writer call")
.expect("stalling writer should signal entry");
tokio::time::timeout(Duration::from_secs(1), final_block)
.await
.expect("producer should supply the block whose send fills the queue")
.expect("reader should signal final block");
let expected_queued_bytes =
u64::try_from((inflight_blocks + 1) * BLOCK_SIZE).expect("queued byte count should fit the gauge");
tokio::time::timeout(Duration::from_secs(1), async {
while rustfs_io_metrics::current_ec_encode_inflight_bytes() < gauge_baseline + expected_queued_bytes {
tokio::task::yield_now().await;
}
})
.await
.expect("producer should account for the pending send after the queue fills");
encode.abort();
assert!(matches!(encode.await, Err(err) if err.is_cancelled()), "encode task should be cancelled");
tokio::time::timeout(Duration::from_secs(1), reader_dropped)
.await
.expect("cancelling a full queue should abort its producer")
.expect("full-queue producer should signal reader drop");
assert_eq!(
writes.load(std::sync::atomic::Ordering::SeqCst),
1,
"cancellation must not resume the stalled writer"
);
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
gauge_baseline,
"cancelling a full queue must settle queued and pending bytes"
);
}
#[tokio::test]
#[serial_test::serial]
async fn cancelling_vec_encode_drops_blocked_producer() {
aborting_encode_drops_blocked_producer(EncodePipeline::Vec).await;
}
#[tokio::test]
#[serial_test::serial]
async fn cancelling_bytesmut_encode_drops_blocked_producer() {
aborting_encode_drops_blocked_producer(EncodePipeline::BytesMut).await;
}
#[tokio::test]
#[serial_test::serial]
async fn cancelling_batched_encode_drops_blocked_producer() {
aborting_encode_drops_blocked_producer(EncodePipeline::Batched).await;
}
#[tokio::test]
#[serial_test::serial]
async fn vec_writer_error_aborts_blocked_producer() {
writer_error_aborts_blocked_producer(EncodePipeline::Vec).await;
}
#[tokio::test]
#[serial_test::serial]
async fn bytesmut_writer_error_aborts_blocked_producer() {
writer_error_aborts_blocked_producer(EncodePipeline::BytesMut).await;
}
#[tokio::test]
#[serial_test::serial]
async fn batched_writer_error_aborts_blocked_producer() {
writer_error_aborts_blocked_producer(EncodePipeline::Batched).await;
}
#[tokio::test]
#[serial_test::serial]
async fn cancelling_full_queue_settles_pending_send() {
aborting_full_queue_settles_pending_send().await;
}
#[tokio::test]
async fn helper_writers_cover_flush_and_shutdown_paths() {
let mut failing_write = FailingWriteWriter;
@@ -1733,99 +1329,25 @@ mod tests {
}
#[tokio::test]
#[serial_test::serial]
async fn queued_inflight_bytes_are_settled_on_all_queue_exit_paths() {
let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
let (tx, rx) = mpsc::channel(1);
let queued = vec![Bytes::from_static(b"queued")];
let queued_bytes = queued_block_bytes(&queued);
async fn drain_queued_inflight_bytes_consumes_pending_blocks() {
let (tx, mut rx) = mpsc::channel(2);
tx.send(vec![Bytes::from_static(b"queued")]).await.unwrap();
drop(tx);
send_queued(&tx, queued, queued_bytes)
.await
.expect("first queue entry should fit");
drain_queued_inflight_bytes(&mut rx).await;
let blocked = vec![Bytes::from_static(b"blocked")];
let blocked_bytes = queued_block_bytes(&blocked);
{
let pending_send = send_queued(&tx, blocked, blocked_bytes);
tokio::pin!(pending_send);
assert!(
futures::poll!(pending_send.as_mut()).is_pending(),
"full queue must suspend producer send"
);
}
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
baseline + u64::try_from(queued_bytes).expect("queue bytes fit the gauge"),
"dropping a pending producer send must compensate its bytes"
);
drop(rx);
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
baseline,
"dropping the receiver must settle every buffered entry"
);
let rejected = vec![Bytes::from_static(b"rejected")];
let rejected_bytes = queued_block_bytes(&rejected);
assert!(
send_queued(&tx, rejected, rejected_bytes).await.is_err(),
"closed receiver must reject a new send"
);
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
baseline,
"failed sends must compensate their bytes"
);
assert!(rx.recv().await.is_none());
}
#[tokio::test]
#[serial_test::serial]
async fn queued_batch_entry_settles_bytes_before_handoff() {
let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
let (tx, rx) = mpsc::channel(2);
let mut rx = rx;
let batch = vec![vec![Bytes::from_static(b"queued")], vec![Bytes::from_static(b"batch")]];
let batch_bytes = queued_batch_bytes(&batch);
async fn drain_queued_batched_inflight_bytes_consumes_pending_batches() {
let (tx, mut rx) = mpsc::channel(2);
tx.send(vec![vec![Bytes::from_static(b"queued")]]).await.unwrap();
drop(tx);
send_queued(&tx, batch, batch_bytes).await.expect("batch should be queued");
let batch = rx.recv().await.expect("queued batch should be received").into_inner();
assert_eq!(batch_bytes, queued_batch_bytes(&batch));
drain_queued_batched_inflight_bytes(&mut rx).await;
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
baseline,
"receiving a batch must settle all contained block bytes before shard writes"
);
}
#[tokio::test]
#[serial_test::serial]
async fn queued_entry_settles_when_a_closed_receiver_accepts_an_outstanding_permit() {
let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
let (tx, mut rx) = mpsc::channel(1);
let permit = tx
.clone()
.reserve_owned()
.await
.expect("open receiver should reserve queue capacity");
rx.close();
assert!(
matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
"an outstanding permit must leave the closed queue observably empty"
);
let block = vec![Bytes::from_static(b"late-permit")];
let block_bytes = queued_block_bytes(&block);
permit.send(InflightEntry::new(block, block_bytes));
drop(rx);
assert_eq!(
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
baseline,
"a queue entry sent through an outstanding permit must settle when Tokio drops it"
);
assert!(rx.recv().await.is_none());
}
#[tokio::test]
+5 -5
View File
@@ -640,7 +640,7 @@ impl Erasure {
/// # Returns
/// A vector of encoded shards as `Bytes`.
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
#[hotpath::measure(impl_type = "Erasure")]
#[hotpath::measure]
pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
@@ -688,7 +688,7 @@ impl Erasure {
/// Encode owned data, avoiding a copy when the caller already has a heap buffer.
/// Falls back to copying into a new buffer if zero-copy conversion fails.
#[hotpath::measure(impl_type = "Erasure")]
#[hotpath::measure]
pub fn encode_data_owned(&self, data: Vec<u8>) -> io::Result<Vec<Bytes>> {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
@@ -752,7 +752,7 @@ impl Erasure {
/// block), the `resize(need_total_size)` below stays within capacity for every
/// `data_len <= block_size` — both shard-size formulas are monotone in
/// `data_len` — so this function never reallocates the buffer.
#[hotpath::measure(impl_type = "Erasure")]
#[hotpath::measure]
pub fn encode_data_bytes_mut(&self, mut data_buffer: BytesMut, data_len: usize) -> io::Result<Vec<Bytes>> {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
@@ -805,7 +805,7 @@ impl Erasure {
///
/// # Returns
/// Ok if reconstruction succeeds, error otherwise.
#[hotpath::measure(impl_type = "Erasure")]
#[hotpath::measure]
pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
if self.parity_shards > 0 {
if self.uses_legacy {
@@ -825,7 +825,7 @@ impl Erasure {
}
/// Decode and reconstruct missing data shards, then regenerate parity shards.
#[hotpath::measure(impl_type = "Erasure")]
#[hotpath::measure]
pub fn decode_data_and_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
if self.parity_shards > 0 {
if self.uses_legacy {
+22 -409
View File
@@ -19,8 +19,6 @@ use crate::diagnostics::get::{
GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
};
#[cfg(feature = "hotpath")]
use crate::disk::FileWriter;
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
use crate::erasure::coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
use bytes::Bytes;
@@ -38,11 +36,6 @@ use std::time::Instant;
use tokio::io::{AsyncRead, ReadBuf};
use tracing::debug;
#[cfg(all(test, feature = "hotpath"))]
tokio::task_local! {
static FORCE_MMAP_COPY_FAILURE_FOR_TEST: ();
}
/// A shard source for the bitrot reader.
///
/// `InMemory` keeps the `Bytes` concrete instead of erasing it behind
@@ -367,21 +360,10 @@ async fn open_disk_reader(
mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
});
let mmap_result = {
#[cfg(all(test, feature = "hotpath"))]
if FORCE_MMAP_COPY_FAILURE_FOR_TEST.try_with(|_| ()).is_ok() {
Err(DiskError::other("forced mmap-copy failure for test"))
} else {
disk.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
.await
}
#[cfg(not(all(test, feature = "hotpath")))]
{
disk.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
.await
}
};
match mmap_result {
match disk
.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
.await
{
Ok(bytes) => {
let duration_ms = zero_copy_start.elapsed().as_secs_f64() * 1000.0;
@@ -416,11 +398,7 @@ async fn open_disk_reader(
}
return match stream_result {
Ok(reader) => {
#[cfg(feature = "hotpath")]
let reader = instrument_raw_shard_reader(reader, disk.is_local());
Ok(wrap_first_read_metrics(reader, metrics_path))
}
Ok(reader) => Ok(wrap_first_read_metrics(reader, metrics_path)),
Err(_) => Err(err),
};
}
@@ -429,8 +407,6 @@ async fn open_disk_reader(
let stream_start = stage_metrics_enabled.then(Instant::now);
let reader = disk.read_file_stream(bucket, path, offset, length).await?;
#[cfg(feature = "hotpath")]
let reader = instrument_raw_shard_reader(reader, disk.is_local());
if let Some(metrics_path) = metrics_path {
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_STREAM, stream_start);
}
@@ -451,37 +427,6 @@ fn wrap_first_read_metrics(reader: FileReader, metrics_path: Option<&'static str
ShardReader::Stream(reader)
}
// The labels are deliberately fixed: object keys, disk paths, and remote hosts
// are all high-cardinality or sensitive and belong nowhere in a profiling report.
#[cfg(feature = "hotpath")]
const RAW_SHARD_READ_LOCAL_LABEL: &str = "EC raw shard read local";
#[cfg(feature = "hotpath")]
const RAW_SHARD_READ_REMOTE_LABEL: &str = "EC raw shard read remote";
#[cfg(feature = "hotpath")]
const RAW_SHARD_WRITE_LOCAL_LABEL: &str = "EC raw shard write local";
#[cfg(feature = "hotpath")]
const RAW_SHARD_WRITE_REMOTE_LABEL: &str = "EC raw shard write remote";
#[cfg(feature = "hotpath")]
fn instrument_raw_shard_reader(reader: FileReader, is_local: bool) -> FileReader {
// `io!` aggregates by call site, so the local and remote branches must remain distinct.
if is_local {
Box::new(hotpath::io!(reader, label = RAW_SHARD_READ_LOCAL_LABEL))
} else {
Box::new(hotpath::io!(reader, label = RAW_SHARD_READ_REMOTE_LABEL))
}
}
#[cfg(feature = "hotpath")]
fn instrument_raw_shard_writer(writer: FileWriter, is_local: bool) -> FileWriter {
// `io!` aggregates by call site, so the local and remote branches must remain distinct.
if is_local {
Box::new(hotpath::io!(writer, label = RAW_SHARD_WRITE_LOCAL_LABEL))
} else {
Box::new(hotpath::io!(writer, label = RAW_SHARD_WRITE_REMOTE_LABEL))
}
}
fn bitrot_encoded_range(offset: usize, length: usize, shard_size: usize, checksum_algo: HashAlgorithm) -> (usize, usize) {
(
offset.div_ceil(shard_size) * checksum_algo.size() + offset,
@@ -741,8 +686,6 @@ pub async fn create_bitrot_writer(
};
let file = disk.create_file("", volume, path, length).await?;
#[cfg(feature = "hotpath")]
let file = instrument_raw_shard_writer(file, disk.is_local());
CustomWriter::new_tokio_writer(file)
} else {
return Err(DiskError::DiskNotFound);
@@ -755,182 +698,6 @@ pub async fn create_bitrot_writer(
mod tests {
use super::*;
#[cfg(feature = "hotpath")]
use crate::cluster::rpc::RemoteDisk;
#[cfg(feature = "hotpath")]
use crate::cluster::rpc::internode_data_transport::{
InternodeDataTransport, InternodeDataTransportCapabilities, ReadStreamRequest, WalkDirStreamRequest, WriteStreamRequest,
};
#[cfg(feature = "hotpath")]
use crate::disk::{Disk, DiskOption, error::Result};
#[cfg(feature = "hotpath")]
#[derive(Debug, Clone, Default)]
struct TestRemoteDataTransport {
bytes: Arc<Mutex<Vec<u8>>>,
}
#[cfg(feature = "hotpath")]
impl TestRemoteDataTransport {
fn bytes(&self) -> Vec<u8> {
self.bytes
.lock()
.expect("test remote transport bytes lock should not be poisoned")
.clone()
}
}
#[cfg(feature = "hotpath")]
#[derive(Debug)]
struct TestRemoteWriter {
bytes: Arc<Mutex<Vec<u8>>>,
}
#[cfg(feature = "hotpath")]
impl tokio::io::AsyncWrite for TestRemoteWriter {
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
self.bytes
.lock()
.expect("test remote transport bytes lock should not be poisoned")
.extend_from_slice(buf);
Poll::Ready(Ok(buf.len()))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Poll::Ready(Ok(()))
}
}
#[cfg(feature = "hotpath")]
#[async_trait::async_trait]
impl InternodeDataTransport for TestRemoteDataTransport {
async fn open_read(&self, _request: ReadStreamRequest) -> Result<FileReader> {
Ok(Box::new(Cursor::new(self.bytes())))
}
async fn open_write(&self, _request: WriteStreamRequest) -> Result<FileWriter> {
Ok(Box::new(TestRemoteWriter {
bytes: Arc::clone(&self.bytes),
}))
}
async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result<FileReader> {
panic!("open_walk_dir must not be used by the raw shard I/O test")
}
fn name(&self) -> &'static str {
"bitrot-test-remote"
}
fn capabilities(&self) -> InternodeDataTransportCapabilities {
InternodeDataTransportCapabilities::tcp_http()
}
}
async fn local_test_disk() -> (DiskStore, tempfile::TempDir) {
use crate::disk::endpoint::Endpoint;
use crate::disk::{DiskOption, new_disk};
let dir = tempfile::tempdir().expect("tempdir should be created");
let mut endpoint =
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("local disk should be created");
(disk, dir)
}
#[cfg(feature = "hotpath")]
async fn remote_test_disk() -> (DiskStore, TestRemoteDataTransport) {
use crate::disk::endpoint::Endpoint;
let endpoint = Endpoint {
url: url::Url::parse("http://remote-node:9000/data/rustfs0").expect("test remote endpoint should parse"),
is_local: false,
pool_idx: 0,
set_idx: 0,
disk_idx: 0,
};
let transport = TestRemoteDataTransport::default();
let remote = RemoteDisk::new(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
Arc::new(transport.clone()),
)
.await
.expect("test remote disk should be created");
(Arc::new(Disk::Remote(Box::new(remote))), transport)
}
async fn round_trip_disk_bitrot(disk: &DiskStore, bucket: &str, path: &str, payload: &[u8], shard_size: usize) -> Vec<u8> {
disk.make_volume(bucket).await.expect("volume should be created");
let mut writer = create_bitrot_writer(
false,
Some(disk),
bucket,
path,
i64::try_from(payload.len()).expect("test payload length should fit i64"),
shard_size,
HashAlgorithm::None,
)
.await
.expect("disk bitrot writer should open the raw shard file");
for chunk in payload.chunks(shard_size) {
writer
.write(chunk)
.await
.expect("disk bitrot writer should preserve each shard block");
}
writer.shutdown().await.expect("disk bitrot writer should close cleanly");
let mut reader = create_bitrot_reader(
None,
Some(disk),
bucket,
path,
0,
payload.len(),
shard_size,
HashAlgorithm::None,
false,
false,
)
.await
.expect("disk bitrot reader should open the raw shard file")
.expect("disk bitrot reader should exist");
let mut actual = Vec::with_capacity(payload.len());
while actual.len() < payload.len() {
let remaining = payload.len() - actual.len();
let mut chunk = vec![0; remaining.min(shard_size)];
let read = reader
.read(&mut chunk)
.await
.expect("disk bitrot reader should return the complete shard body");
assert!(read > 0, "disk bitrot reader must not end before the expected shard body is complete");
actual.extend_from_slice(&chunk[..read]);
}
actual
}
#[test]
fn object_mmap_read_enabled_accepts_legacy_zero_copy_alias() {
temp_env::with_vars(
@@ -957,165 +724,6 @@ mod tests {
);
}
#[cfg(feature = "hotpath")]
#[test]
fn raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes() {
const CHILD_ENV: &str = "RUSTFS_HOTPATH_RAW_SHARD_IO_TEST_CHILD";
if std::env::var_os(CHILD_ENV).is_none() {
let status = std::process::Command::new(std::env::current_exe().expect("test executable path should be available"))
.arg("--exact")
.arg("io_support::bitrot::tests::raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes")
.arg("--nocapture")
.env(CHILD_ENV, "1")
.status()
.expect("isolated HotPath I/O test process should start");
assert!(status.success(), "isolated HotPath I/O test process should pass");
return;
}
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("test runtime should be created")
.block_on(raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes_in_isolated_process());
}
#[cfg(feature = "hotpath")]
async fn raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes_in_isolated_process() {
use hotpath::{Format, HotpathGuardBuilder, Section};
use tokio::io::AsyncReadExt;
let report_dir = tempfile::tempdir().expect("report tempdir should be created");
let report_path = report_dir.path().join("hotpath-io.json");
let guard = HotpathGuardBuilder::new("raw_shard_io_test")
.format(Format::Json)
.output_path(&report_path)
.sections(vec![Section::Io])
.build();
let (disk, _dir) = local_test_disk().await;
let bucket = "test-bucket";
let path = "obj/hotpath-part.1";
let payload = b"local shard bytes";
let shard_size = 4;
let local_read = round_trip_disk_bitrot(&disk, bucket, path, payload, shard_size).await;
assert_eq!(local_read, payload, "local raw shard I/O instrumentation must not alter stored bytes");
let fallback_path = "obj/hotpath-mmap-fallback-part.1";
let fallback_payload = b"mmap fallback shard bytes";
disk.write_all("test-bucket", fallback_path, Bytes::from_static(fallback_payload))
.await
.expect("fallback shard file should be written");
let mut fallback_reader = FORCE_MMAP_COPY_FAILURE_FOR_TEST
.scope((), open_disk_reader(&disk, bucket, fallback_path, 0, fallback_payload.len(), true, None))
.await
.expect("mmap-copy failure should fall back to a raw shard stream");
assert!(
matches!(fallback_reader, ShardReader::Stream(_)),
"forced mmap-copy failure must use the streaming fallback"
);
let mut fallback_read = Vec::new();
fallback_reader
.read_to_end(&mut fallback_read)
.await
.expect("mmap-copy fallback stream should preserve bytes");
assert_eq!(fallback_read, fallback_payload);
let (remote_disk, remote_transport) = remote_test_disk().await;
let remote_payload = b"remote shard bytes";
let mut remote_writer = create_bitrot_writer(
false,
Some(&remote_disk),
bucket,
"obj/hotpath-remote-part.1",
i64::try_from(remote_payload.len()).expect("remote payload length should fit i64"),
shard_size,
HashAlgorithm::None,
)
.await
.expect("remote bitrot writer should use the production raw writer path");
for chunk in remote_payload.chunks(shard_size) {
remote_writer
.write(chunk)
.await
.expect("remote bitrot writer should preserve bytes");
}
remote_writer
.shutdown()
.await
.expect("remote bitrot writer should close cleanly");
assert_eq!(remote_transport.bytes(), remote_payload);
let mut reader = create_bitrot_reader(
None,
Some(&remote_disk),
bucket,
"obj/hotpath-remote-part.1",
0,
remote_payload.len(),
shard_size,
HashAlgorithm::None,
false,
true,
)
.await
.expect("remote bitrot reader should use the production raw reader path")
.expect("remote bitrot reader should exist");
let mut remote_read = Vec::with_capacity(remote_payload.len());
while remote_read.len() < remote_payload.len() {
let remaining = remote_payload.len() - remote_read.len();
let mut chunk = vec![0; remaining.min(shard_size)];
let read = reader
.read(&mut chunk)
.await
.expect("remote bitrot reader should preserve bytes");
assert!(read > 0, "remote bitrot reader must not end before the expected shard body is complete");
remote_read.extend_from_slice(&chunk[..read]);
}
assert_eq!(remote_read, remote_payload);
drop(guard);
let report = std::fs::read_to_string(&report_path).expect("HotPath I/O report should be written");
let report: serde_json::Value = serde_json::from_str(&report).expect("HotPath I/O report should be valid JSON");
let entries = report["io"]["data"]
.as_array()
.expect("HotPath I/O report should include data rows");
let io_bytes = |label: &str, direction: &str| {
let byte_count: u64 = entries
.iter()
.filter(|entry| entry["label"].as_str().is_some_and(|entry_label| entry_label == label))
.filter_map(|entry| entry[direction]["bytes"].as_u64())
.sum();
assert!(byte_count > 0, "report must include fixed label {label}");
byte_count
};
let payload_len = u64::try_from(payload.len()).expect("test payload length should fit u64");
assert_eq!(
io_bytes(RAW_SHARD_READ_LOCAL_LABEL, "read"),
payload_len + u64::try_from(fallback_payload.len()).expect("fallback payload length should fit u64")
);
assert_eq!(io_bytes(RAW_SHARD_WRITE_LOCAL_LABEL, "write"), payload_len);
assert_eq!(
io_bytes(RAW_SHARD_READ_REMOTE_LABEL, "read"),
u64::try_from(remote_payload.len()).expect("remote payload length should fit u64")
);
assert_eq!(
io_bytes(RAW_SHARD_WRITE_REMOTE_LABEL, "write"),
u64::try_from(remote_payload.len()).expect("remote payload length should fit u64")
);
for label in [
RAW_SHARD_READ_LOCAL_LABEL,
RAW_SHARD_READ_REMOTE_LABEL,
RAW_SHARD_WRITE_LOCAL_LABEL,
RAW_SHARD_WRITE_REMOTE_LABEL,
] {
assert!(
!label.contains(['/', ':', '?', '@']),
"raw shard I/O labels must not carry a path, host, query, or credential delimiter: {label}"
);
}
}
#[test]
fn object_mmap_read_max_length_defaults_and_env_override() {
temp_env::with_var(ENV_OBJECT_MMAP_READ_MAX_LENGTH, None::<&str>, || {
@@ -1133,9 +741,25 @@ mod tests {
// be materialized in memory by the mmap-copy path; over-cap reads stream.
#[tokio::test]
async fn open_disk_reader_streams_when_length_exceeds_mmap_cap() {
use crate::disk::endpoint::Endpoint;
use crate::disk::{DiskOption, new_disk};
use tokio::io::AsyncReadExt;
let (disk, _dir) = local_test_disk().await;
let dir = tempfile::tempdir().expect("tempdir should be created");
let mut endpoint =
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("local disk should be created");
let payload = vec![7u8; 4096];
disk.make_volume("test-bucket").await.expect("volume should be created");
@@ -1190,17 +814,6 @@ mod tests {
.await;
}
#[tokio::test]
async fn disk_bitrot_reader_and_writer_preserve_full_shard_body() {
let (disk, _dir) = local_test_disk().await;
let bucket = "test-bucket";
let path = "obj/wrapped-part.1";
let payload = b"wrapped shard body";
let actual = round_trip_disk_bitrot(&disk, bucket, path, payload, 4).await;
assert_eq!(actual, payload, "raw shard I/O instrumentation must not alter stored bytes");
}
#[tokio::test]
async fn test_create_bitrot_reader_with_inline_data() {
let test_data = b"hello world test data";
@@ -337,15 +337,6 @@ pub struct NotificationPeerErr {
pub err: Option<Error>,
}
/// One peer's answer to a KMS configuration fingerprint probe.
pub struct PeerKmsConfigFingerprint {
pub host: String,
/// `None` when the peer has no KMS configuration of its own, or could not
/// be asked at all, in which case `err` carries the reason.
pub fingerprint: Option<String>,
pub err: Option<Error>,
}
fn notification_peer_result<T>(host: String, result: Result<T>) -> NotificationPeerErr {
NotificationPeerErr { host, err: result.err() }
}
@@ -527,47 +518,6 @@ impl NotificationSys {
self.signal_dynamic_config(sub_sys, false).await
}
/// Ask every peer to re-read the cluster-persisted KMS configuration.
///
/// Best-effort by contract: the caller has already switched locally, so a
/// peer that fails is reported rather than rolled back. Peers built before
/// the KMS subsystem existed reject the signal with an explicit error.
pub async fn reload_kms_config(&self) -> Vec<NotificationPeerErr> {
self.reload_dynamic_config(crate::cluster::rpc::KMS_SIGNAL_SUBSYSTEM).await
}
/// Collect the KMS configuration fingerprint each peer is running.
///
/// A peer whose build predates the KMS subsystem rejects the probe, so it
/// is reported as an error rather than silently agreeing with this node.
pub async fn kms_config_fingerprints(&self) -> Vec<PeerKmsConfigFingerprint> {
let mut futures = Vec::with_capacity(self.peer_clients.len());
for client in self.peer_clients.iter() {
futures.push(async move {
let Some(client) = client else {
return PeerKmsConfigFingerprint {
host: String::new(),
fingerprint: None,
err: Some(Error::other("peer is not reachable")),
};
};
match client.kms_config_fingerprint().await {
Ok(fingerprint) => PeerKmsConfigFingerprint {
host: client.host.to_string(),
fingerprint,
err: None,
},
Err(e) => PeerKmsConfigFingerprint {
host: client.host.to_string(),
fingerprint: None,
err: Some(e),
},
}
});
}
join_all(futures).await
}
pub async fn refresh_config_snapshot(&self) -> Vec<NotificationPeerErr> {
let mut futures = Vec::with_capacity(self.peer_clients.len());
for client in self.peer_clients.iter() {
+6 -6
View File
@@ -199,7 +199,7 @@ impl SetDisks {
);
}
#[hotpath::measure(impl_type = "SetDisks")]
#[hotpath::measure]
pub async fn read_version_optimized(
&self,
bucket: &str,
@@ -238,7 +238,7 @@ impl SetDisks {
}
#[tracing::instrument(level = "debug", skip(self))]
#[hotpath::measure(impl_type = "SetDisks")]
#[hotpath::measure]
pub(super) async fn get_object_fileinfo(
&self,
bucket: &str,
@@ -410,7 +410,7 @@ impl SetDisks {
Ok((fi, parts_metadata, op_online_disks))
}
#[hotpath::measure(impl_type = "SetDisks")]
#[hotpath::measure]
pub(super) async fn get_object_info_and_quorum(
&self,
bucket: &str,
@@ -605,7 +605,7 @@ impl SetDisks {
}
#[allow(clippy::too_many_arguments)]
#[hotpath::measure(impl_type = "SetDisks")]
#[hotpath::measure]
pub(super) async fn get_object_with_fileinfo<W>(
// &self,
bucket: &str,
@@ -1140,7 +1140,7 @@ impl SetDisks {
}
#[allow(clippy::too_many_arguments)]
#[hotpath::measure(impl_type = "SetDisks")]
#[hotpath::measure]
pub(super) async fn get_object_decode_reader_with_fileinfo(
bucket: &str,
object: &str,
@@ -1296,7 +1296,7 @@ impl SetDisks {
}
#[allow(clippy::too_many_arguments)]
#[hotpath::measure(impl_type = "SetDisks")]
#[hotpath::measure]
async fn build_codec_streaming_part_reader(
bucket: &str,
object: &str,
+1 -1
View File
@@ -238,7 +238,7 @@ impl ECStore {
}
#[instrument(skip(self, data))]
#[hotpath::measure(impl_type = "ECStore")]
#[hotpath::measure]
pub(super) async fn handle_put_object_part(
&self,
bucket: &str,
+2 -2
View File
@@ -815,7 +815,7 @@ impl ECStore {
}
#[instrument(level = "debug", skip(self))]
#[hotpath::measure(impl_type = "ECStore")]
#[hotpath::measure]
pub(super) async fn handle_get_object_reader(
&self,
bucket: &str,
@@ -849,7 +849,7 @@ impl ECStore {
}
#[instrument(level = "debug", skip(self, data))]
#[hotpath::measure(impl_type = "ECStore")]
#[hotpath::measure]
pub(super) async fn handle_put_object(
&self,
bucket: &str,
@@ -1,95 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
const STORE_MULTIPART: &str = include_str!("../src/store/multipart.rs");
const STORE_OBJECT: &str = include_str!("../src/store/object.rs");
const ERASURE: &str = include_str!("../src/erasure/coding/erasure.rs");
const DECODE: &str = include_str!("../src/erasure/coding/decode.rs");
const ENCODE: &str = include_str!("../src/erasure/coding/encode.rs");
const LOCAL_DISK: &str = include_str!("../src/disk/local.rs");
const BITROT: &str = include_str!("../src/erasure/coding/bitrot.rs");
const SET_DISK_READ: &str = include_str!("../src/set_disk/read.rs");
fn assert_measured_as(source: &str, impl_type: &str, function: &str) {
let attribute = format!("#[hotpath::measure(impl_type = \"{impl_type}\")]\n");
let function_start = format!("fn {function}");
let mut remaining = source;
while let Some(attribute_offset) = remaining.find(&attribute) {
let measured_source = &remaining[attribute_offset + attribute.len()..];
if measured_source.find(&function_start).is_some_and(|offset| offset < 256) {
return;
}
remaining = measured_source;
}
panic!("missing HotPath impl_type={impl_type} for {function}");
}
#[test]
fn inherent_hotpath_measurements_have_cpu_attribution_types() {
assert_measured_as(STORE_MULTIPART, "ECStore", "handle_put_object_part");
for function in ["handle_get_object_reader", "handle_put_object"] {
assert_measured_as(STORE_OBJECT, "ECStore", function);
}
for function in [
"encode_data",
"encode_data_owned",
"encode_data_bytes_mut",
"decode_data",
"decode_data_and_parity",
] {
assert_measured_as(ERASURE, "Erasure", function);
}
assert_measured_as(DECODE, "ParallelReader", "read");
assert_measured_as(DECODE, "Erasure", "decode");
for function in [
"encode",
"encode_batched",
"encode_inline_small",
"encode_single_block_non_inline",
] {
assert_measured_as(ENCODE, "Erasure", function);
}
for function in ["read_metadata_with_dmtime", "read_all_data"] {
assert_measured_as(LOCAL_DISK, "LocalDisk", function);
}
assert_measured_as(BITROT, "BitrotReader", "read");
assert!(
BITROT.contains("#[hotpath::measure(label = \"BitrotWriter::write\", impl_type = \"BitrotWriter\")]"),
"BitrotWriter::write must retain its stable label and CPU impl_type",
);
for function in [
"read_version_optimized",
"get_object_fileinfo",
"get_object_info_and_quorum",
"get_object_with_fileinfo",
"get_object_decode_reader_with_fileinfo",
"build_codec_streaming_part_reader",
] {
assert_measured_as(SET_DISK_READ, "SetDisks", function);
}
}
#[test]
fn module_hotpath_measurements_remain_untyped() {
assert!(
BITROT.contains("#[hotpath::measure]\npub async fn bitrot_verify"),
"bitrot_verify is a module function and must not claim an impl type",
);
assert!(
SET_DISK_READ.contains("#[hotpath::measure]\nasync fn setup_multipart_part_readers"),
"setup_multipart_part_readers is a module function and must not claim an impl type",
);
}
+30 -239
View File
@@ -41,7 +41,7 @@ use rustfs_policy::policy::Args;
use rustfs_policy::policy::opa;
use rustfs_policy::policy::{Policy, PolicyDoc, iam_policy_claim_name_sa, policy_needs_existing_object_tag_for_args};
use serde_json::Value;
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::OnceLock;
use time::OffsetDateTime;
@@ -61,51 +61,10 @@ const VERIFIED_FEDERATED_POLICY_CLAIM: &str = "x-rustfs-internal-federated-polic
const FEDERATED_POLICY_BOUNDARY_CLAIM: &str = "x-rustfs-internal-federated-boundary";
pub(crate) const SITE_REPLICATOR_CLAIM: &str = "site-replicator";
#[derive(Clone)]
enum PolicyPluginState {
Disabled,
Initializing,
Ready(opa::AuthZPlugin),
Failed,
}
static POLICY_PLUGIN_CLIENT: OnceLock<Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>>> = OnceLock::new();
static POLICY_PLUGIN_STATE: OnceLock<Arc<RwLock<PolicyPluginState>>> = OnceLock::new();
fn get_policy_plugin_state() -> Arc<RwLock<PolicyPluginState>> {
POLICY_PLUGIN_STATE
.get_or_init(|| {
let configured = opa::is_configured();
let state = Arc::new(RwLock::new(if configured {
PolicyPluginState::Initializing
} else {
PolicyPluginState::Disabled
}));
if configured {
let state = Arc::clone(&state);
tokio::spawn(async move {
let next_state = match opa::lookup_config().await {
Ok(conf) if conf.enable() => {
info!("OPA plugin enabled");
PolicyPluginState::Ready(opa::AuthZPlugin::new(conf))
}
Ok(_) => PolicyPluginState::Failed,
Err(e) => {
error!(
component = "iam",
subsystem = "policy_plugin",
result = "configuration_load_failed",
error_kind = e.kind(),
"OPA plugin configuration load failed"
);
PolicyPluginState::Failed
}
};
*state.write().await = next_state;
});
}
state
})
.clone()
fn get_policy_plugin_client() -> Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>> {
POLICY_PLUGIN_CLIENT.get_or_init(|| Arc::new(RwLock::new(None))).clone()
}
pub struct IamSys<T> {
@@ -214,7 +173,19 @@ impl<T: Store> IamSys<T> {
/// # Returns
/// A new instance of IamSys
pub fn new(store: Arc<IamCache<T>>) -> Self {
get_policy_plugin_state();
tokio::spawn(async move {
match opa::lookup_config().await {
Ok(conf) => {
if conf.enable() {
Self::set_policy_plugin_client(opa::AuthZPlugin::new(conf)).await;
info!("OPA plugin enabled");
}
}
Err(e) => {
error!("Error loading OPA configuration err:{}", e);
}
};
});
Self {
store,
@@ -235,22 +206,15 @@ impl<T: Store> IamSys<T> {
}
pub async fn set_policy_plugin_client(client: rustfs_policy::policy::opa::AuthZPlugin) {
let policy_plugin_state = get_policy_plugin_state();
let mut guard = policy_plugin_state.write().await;
*guard = PolicyPluginState::Ready(client);
let policy_plugin_client = get_policy_plugin_client();
let mut guard = policy_plugin_client.write().await;
*guard = Some(client);
}
pub async fn get_policy_plugin_client() -> Option<rustfs_policy::policy::opa::AuthZPlugin> {
let policy_plugin_state = get_policy_plugin_state();
let guard = policy_plugin_state.read().await;
match &*guard {
PolicyPluginState::Ready(client) => Some(client.clone()),
PolicyPluginState::Disabled | PolicyPluginState::Initializing | PolicyPluginState::Failed => None,
}
}
async fn policy_plugin_state() -> PolicyPluginState {
get_policy_plugin_state().read().await.clone()
let policy_plugin_client = get_policy_plugin_client();
let guard = policy_plugin_client.read().await;
guard.clone()
}
pub async fn load_group(&self, name: &str) -> Result<()> {
@@ -1096,20 +1060,11 @@ impl<T: Store> IamSys<T> {
};
}
match Self::policy_plugin_state().await {
PolicyPluginState::Ready(_) => {
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::Opa,
};
}
PolicyPluginState::Initializing | PolicyPluginState::Failed => {
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::Deny,
};
}
PolicyPluginState::Disabled => {}
if Self::get_policy_plugin_client().await.is_some() {
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::Opa,
};
}
let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
@@ -1208,22 +1163,7 @@ impl<T: Store> IamSys<T> {
};
}
let (resolved_policies, combined_policy) = self.store.merge_policies(&policies.join(",")).await;
if args.deny_only {
let resolved_policy_names: HashSet<&str> = resolved_policies
.split(',')
.filter(|policy_name| !policy_name.trim().is_empty())
.collect();
if policies
.iter()
.any(|policy_name| !resolved_policy_names.contains(policy_name.as_str()))
{
return PreparedIamAuth {
needs_existing_object_tag: false,
mode: PreparedIamMode::Deny,
};
}
}
let combined_policy = self.get_combined_policy(&policies).await;
PreparedIamAuth {
needs_existing_object_tag: policy_needs_existing_object_tag_for_args(&combined_policy, args).await,
mode: PreparedIamMode::Regular { combined_policy },
@@ -1763,7 +1703,7 @@ mod tests {
fn deprecated_list_polices_api_is_available() {
let _ = IamSys::<StsTestMockStore>::list_polices;
}
use rustfs_policy::policy::action::{Action, AdminAction, S3Action, StsAction};
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
use rustfs_policy::policy::policy_uses_existing_object_tag_conditions;
use serde_json::Value;
use std::{
@@ -3369,155 +3309,6 @@ mod tests {
assert!(!iam_sys.eval_prepared(&prepared, &args).await);
}
#[tokio::test]
async fn regular_deny_only_rejects_unresolved_mapped_policy() {
let iam_sys = test_iam_sys().await;
let user = "regular-unresolved-policy";
let identity = UserIdentity::from(Credentials {
access_key: user.to_string(),
secret_key: "longenoughsecret".to_string(),
status: ACCOUNT_ON.to_string(),
..Default::default()
});
iam_sys.store.cache.with_write_lock(|cache| {
let now = OffsetDateTime::now_utc();
cache.add_or_update_user(user, &identity, now);
cache.add_or_update_user_policy(user, &MappedPolicy::new("readwrite,missing-policy"), now);
});
let groups = None;
let claims = HashMap::new();
let conditions = HashMap::new();
let args = Args {
account: user,
groups: &groups,
action: Action::StsAction(StsAction::AssumeRoleAction),
bucket: "",
conditions: &conditions,
is_owner: false,
object: "",
claims: &claims,
deny_only: true,
};
let prepared = iam_sys.prepare_regular_auth(&args).await;
assert!(matches!(prepared.mode, PreparedIamMode::Deny));
assert!(
!iam_sys.eval_prepared(&prepared, &args).await,
"deny-only evaluation must fail closed when a mapped policy document is unresolved"
);
}
#[tokio::test]
async fn regular_full_evaluation_keeps_resolved_part_of_partial_mapping() {
let iam_sys = test_iam_sys().await;
let user = "regular-partial-policy";
let identity = UserIdentity::from(Credentials {
access_key: user.to_string(),
secret_key: "longenoughsecret".to_string(),
status: ACCOUNT_ON.to_string(),
..Default::default()
});
iam_sys.store.cache.with_write_lock(|cache| {
let now = OffsetDateTime::now_utc();
cache.add_or_update_user(user, &identity, now);
cache.add_or_update_user_policy(user, &MappedPolicy::new("readwrite,missing-policy"), now);
});
let groups = None;
let claims = HashMap::new();
let conditions = HashMap::new();
let args = Args {
account: user,
groups: &groups,
action: Action::S3Action(S3Action::GetObjectAction),
bucket: "bucket",
conditions: &conditions,
is_owner: false,
object: "object",
claims: &claims,
deny_only: false,
};
let prepared = iam_sys.prepare_regular_auth(&args).await;
assert!(matches!(prepared.mode, PreparedIamMode::Regular { .. }));
assert!(
iam_sys.eval_prepared(&prepared, &args).await,
"full evaluation must preserve the historical partial-resolution behavior"
);
}
#[tokio::test]
async fn regular_deny_only_honors_group_derived_explicit_deny() {
let iam_sys = test_iam_sys().await;
let deny_policy =
Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":["sts:AssumeRole"]}]}"#)
.expect("group deny policy should parse");
let now = OffsetDateTime::now_utc();
iam_sys
.store
.cache
.add_or_update_policy_doc("deny-assume-role", &PolicyDoc::new(deny_policy), now);
iam_sys
.store
.cache
.add_or_update_group_policy("testgroup", &MappedPolicy::new("deny-assume-role"), now);
let groups = Some(vec!["testgroup".to_string()]);
let claims = HashMap::new();
let conditions = HashMap::new();
let args = Args {
account: "sts-fallback-test-parent",
groups: &groups,
action: Action::StsAction(StsAction::AssumeRoleAction),
bucket: "",
conditions: &conditions,
is_owner: false,
object: "",
claims: &claims,
deny_only: true,
};
let prepared = iam_sys.prepare_regular_auth(&args).await;
assert!(matches!(prepared.mode, PreparedIamMode::Regular { .. }));
assert!(
!iam_sys.eval_prepared(&prepared, &args).await,
"group-derived explicit Deny must remain authoritative"
);
}
#[tokio::test]
async fn regular_deny_only_rejects_unresolved_group_mapping() {
let iam_sys = test_iam_sys().await;
iam_sys.store.cache.add_or_update_group_policy(
"testgroup",
&MappedPolicy::new("readwrite,missing-policy"),
OffsetDateTime::now_utc(),
);
let groups = Some(vec!["testgroup".to_string()]);
let claims = HashMap::new();
let conditions = HashMap::new();
let args = Args {
account: "sts-fallback-test-parent",
groups: &groups,
action: Action::StsAction(StsAction::AssumeRoleAction),
bucket: "",
conditions: &conditions,
is_owner: false,
object: "",
claims: &claims,
deny_only: true,
};
let prepared = iam_sys.prepare_regular_auth(&args).await;
assert!(matches!(prepared.mode, PreparedIamMode::Deny));
assert!(
!iam_sys.eval_prepared(&prepared, &args).await,
"deny-only evaluation must fail closed for an unresolved group-derived mapping"
);
}
#[tokio::test]
async fn test_sts_claim_policy_ignores_unsafe_and_missing_policy_names() {
let store = StsTestMockStore::new(true);
+3 -7
View File
@@ -2195,22 +2195,18 @@ pub fn record_allocator_memory_observation(backend: &'static str, observation: A
}
}
/// Track encoded bytes in the queue hand-off between erasure encode and disk writers.
///
/// This is queue occupancy, not a per-request or process-RSS memory limit. The
/// erasure encoder settles it on failed/cancelled sends, receiver drop, and
/// consumer hand-off before shard writes begin.
/// Track encoded bytes currently queued between erasure encode and disk writers.
#[inline(always)]
pub fn add_ec_encode_inflight_bytes(bytes: usize) {
let next = EC_ENCODE_INFLIGHT_BYTES.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64;
gauge_set_cached!("rustfs_ec_encode_inflight_bytes_current", next as f64);
gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
}
/// Remove encoded bytes from the tracked erasure encode in-flight gauge.
#[inline(always)]
pub fn remove_ec_encode_inflight_bytes(bytes: usize) {
let next = saturating_sub_atomic(&EC_ENCODE_INFLIGHT_BYTES, bytes as u64);
gauge_set_cached!("rustfs_ec_encode_inflight_bytes_current", next as f64);
gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
}
/// Return the current tracked EC encode in-flight bytes.
-17
View File
@@ -64,10 +64,6 @@ md-5 = { workspace = true }
arc-swap = { workspace = true }
rustfs-utils = { workspace = true }
rustfs-security-governance = { workspace = true }
# `EventName` for KMS audit records. A leaf crate with no rustfs dependencies,
# so the audit sink can live outside this crate without a second, drifting
# copy of the event vocabulary.
rustfs-s3-types = { workspace = true }
# HTTP client for Vault
reqwest = { workspace = true }
@@ -77,16 +73,6 @@ vaultrs = { workspace = true }
rustify = { workspace = true }
tokio-util = { workspace = true }
# AWS KMS backend. Credentials come from the standard aws-config provider chain
# (environment, shared profile, IMDS/container roles); this crate never handles
# AWS credential material itself.
aws-config = { workspace = true }
aws-sdk-kms = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
# SdkError variants and raw HTTP status are needed to classify AWS failures for
# the operation policy's retry decisions.
aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
aws-smithy-types = { workspace = true }
[dev-dependencies]
anyhow = { workspace = true }
# Debugging recorder for asserting emitted metrics in tests.
@@ -96,9 +82,6 @@ tempfile = { workspace = true }
temp-env = { workspace = true }
# "net" backs the scripted loopback Vault used by the policy wiring tests.
tokio = { workspace = true, features = ["net", "test-util"] }
# Replays canned AWS KMS HTTP exchanges so the AWS backend tests stay offline.
aws-smithy-http-client = { workspace = true, default-features = false, features = ["test-util"] }
http = { workspace = true }
[features]
default = []
+3 -5
View File
@@ -227,12 +227,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Step 12: Check cache statistics
println!("12. Checking cache statistics...");
if let Some(stats) = encryption_service.cache_stats().await {
if let Some((hits, misses)) = encryption_service.cache_stats().await {
println!(" ✓ Cache statistics:");
println!(" - Cached entries: {}", stats.entries);
println!(" - Cache hits: {}", stats.hits);
println!(" - Cache misses: {}", stats.misses);
println!(" - Entries evicted: {}\n", stats.evictions);
println!(" - Cache hits: {}", hits);
println!(" - Cache misses: {}\n", misses);
} else {
println!(" - Cache is disabled\n");
}
+3 -5
View File
@@ -263,12 +263,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Step 12: Check cache statistics
println!("12. Checking cache statistics...");
if let Some(stats) = encryption_service.cache_stats().await {
if let Some((hits, misses)) = encryption_service.cache_stats().await {
println!(" ✓ Cache statistics:");
println!(" - Cached entries: {}", stats.entries);
println!(" - Cache hits: {}", stats.hits);
println!(" - Cache misses: {}", stats.misses);
println!(" - Entries evicted: {}\n", stats.evictions);
println!(" - Cache hits: {}", hits);
println!(" - Cache misses: {}\n", misses);
} else {
println!(" - Cache is disabled\n");
}
-11
View File
@@ -418,13 +418,6 @@ pub enum BackendSummary {
/// Configured key identifier
key_id: String,
},
/// AWS KMS backend summary
Aws {
/// Configured region, when pinned instead of resolved by the AWS chain
region: Option<String>,
/// Endpoint override, when set for an emulator or private endpoint
endpoint_url: Option<String>,
},
}
impl From<&KmsConfig> for KmsConfigSummary {
@@ -474,10 +467,6 @@ impl From<&KmsConfig> for KmsConfigSummary {
BackendConfig::Static(static_config) => BackendSummary::Static {
key_id: static_config.key_id.clone(),
},
BackendConfig::Aws(aws_config) => BackendSummary::Aws {
region: aws_config.region.clone(),
endpoint_url: aws_config.endpoint_url.clone(),
},
};
Self {
-423
View File
@@ -1,423 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Audit contract for KMS management operations.
//!
//! [`KmsManager`](crate::manager::KmsManager) builds a [`KmsAuditRecord`] for
//! every management operation it serves and hands it to the installed
//! [`KmsAuditSink`]. No sink is installed by default, so a deployment that
//! does not consume KMS audit records pays nothing beyond the `Option` check.
//!
//! The record is deliberately a KMS-side type rather than an audit-crate
//! entry: keeping the dependency edge out of this crate lets the server map
//! records onto its existing audit pipeline (and its delivery semantics)
//! without KMS having to know that pipeline exists.
//!
//! # Redaction
//!
//! A record has no field that can hold key material, and every value that
//! originates from a caller passes through [`redact_encryption_context`]
//! before it is stored. Adding a field here means re-checking that invariant.
use crate::error::KmsError;
use crate::types::OperationContext;
use rustfs_s3_types::EventName;
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, HashMap};
use std::time::Duration;
use uuid::Uuid;
/// Encryption-context keys that describe *where* an object lives rather than
/// anything secret about it. Their values are recorded verbatim; every other
/// key is reduced to a digest.
const ENCRYPTION_CONTEXT_ALLOWLIST: [&str; 5] = ["bucket", "object", "object_key", "algorithm", "sse_type"];
/// Prefix marking a value that was replaced by a digest of itself.
const DIGEST_PREFIX: &str = "sha256:";
/// Hex characters of the digest that are kept. Enough to correlate repeated
/// values across records without being reversible in practice.
const DIGEST_LEN: usize = 16;
/// A KMS management operation that produces an audit record.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum KmsAuditOperation {
/// Creation of a new master key.
CreateKey,
/// Metadata lookup for a single key.
DescribeKey,
/// Enumeration of keys.
ListKeys,
/// Scheduling a key for deletion after the pending window.
ScheduleKeyDeletion,
/// Cancelling a previously scheduled deletion.
CancelKeyDeletion,
/// Returning a disabled key to service.
EnableKey,
/// Taking a key out of service without destroying it.
DisableKey,
/// Rotating a key to a new version.
RotateKey,
/// Irreversible removal of key material once the pending window expired.
DeleteKey,
}
impl KmsAuditOperation {
/// Stable operation name for audit consumers.
pub fn as_str(&self) -> &'static str {
match self {
KmsAuditOperation::CreateKey => "CreateKey",
KmsAuditOperation::DescribeKey => "DescribeKey",
KmsAuditOperation::ListKeys => "ListKeys",
KmsAuditOperation::ScheduleKeyDeletion => "ScheduleKeyDeletion",
KmsAuditOperation::CancelKeyDeletion => "CancelKeyDeletion",
KmsAuditOperation::EnableKey => "EnableKey",
KmsAuditOperation::DisableKey => "DisableKey",
KmsAuditOperation::RotateKey => "RotateKey",
KmsAuditOperation::DeleteKey => "DeleteKey",
}
}
/// Notification/audit event name carried by records for this operation.
pub fn event_name(&self) -> EventName {
match self {
KmsAuditOperation::CreateKey => EventName::KmsKeyCreated,
KmsAuditOperation::DescribeKey | KmsAuditOperation::ListKeys => EventName::KmsKeyAccessed,
KmsAuditOperation::ScheduleKeyDeletion => EventName::KmsKeyDeletionScheduled,
KmsAuditOperation::CancelKeyDeletion => EventName::KmsKeyDeletionCancelled,
KmsAuditOperation::EnableKey => EventName::KmsKeyEnabled,
KmsAuditOperation::DisableKey => EventName::KmsKeyDisabled,
KmsAuditOperation::RotateKey => EventName::KmsKeyRotated,
KmsAuditOperation::DeleteKey => EventName::KmsKeyDeleted,
}
}
}
/// Whether the audited operation completed.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum KmsAuditOutcome {
/// The operation completed and its effect is durable.
Success,
/// The operation failed; `error_class` carries the reason category.
Failure,
}
impl KmsAuditOutcome {
/// Stable outcome name for audit consumers.
pub fn as_str(&self) -> &'static str {
match self {
KmsAuditOutcome::Success => "success",
KmsAuditOutcome::Failure => "failure",
}
}
}
/// Coarse, stable classification of a failed KMS operation.
///
/// Audit consumers alert on these, so the strings are a wire contract: add
/// new classes rather than renaming existing ones.
pub fn error_class(error: &KmsError) -> &'static str {
match error {
KmsError::ConfigurationError { .. } => "configuration",
KmsError::KeyNotFound { .. } => "key_not_found",
KmsError::InvalidKey { .. } => "invalid_key",
KmsError::CryptographicError { .. } => "cryptographic",
KmsError::BackendError { .. } => "backend",
KmsError::AccessDenied { .. } => "access_denied",
KmsError::KeyAlreadyExists { .. } => "key_already_exists",
KmsError::InvalidOperation { .. } => "invalid_operation",
KmsError::InternalError { .. } => "internal",
KmsError::SerializationError { .. } => "serialization",
KmsError::IoError { .. } => "io",
KmsError::CacheError { .. } => "cache",
KmsError::ValidationError { .. } => "validation",
KmsError::UnsupportedAlgorithm { .. } => "unsupported_algorithm",
KmsError::InvalidKeySize { .. } => "invalid_key_size",
KmsError::ContextMismatch { .. } => "context_mismatch",
KmsError::OperationTimedOut { .. } => "timeout",
KmsError::OperationCancelled { .. } => "cancelled",
KmsError::MaterialMissing { .. } => "material_missing",
KmsError::MaterialCorrupt { .. } => "material_corrupt",
KmsError::MaterialAuthenticationFailed { .. } => "material_authentication_failed",
KmsError::UnsupportedFormatVersion { .. } => "unsupported_format_version",
KmsError::KeyVersionNotFound { .. } => "key_version_not_found",
KmsError::Backup(_) => "backup",
KmsError::UnsupportedCapability { .. } => "unsupported_capability",
KmsError::CredentialsUnavailable { .. } => "credentials_unavailable",
}
}
/// One audited KMS management operation.
///
/// Everything an audit consumer needs to answer "who did what to which key,
/// against which backend, and did it work" — and nothing that could carry key
/// material. See the module docs for the redaction rules.
///
/// Tenant attribution is intentionally absent: multi-tenancy is not modelled
/// yet, and an invented tenant value is worse than a missing one. It will
/// arrive as an entry in [`Self::context`] once tenancy lands.
#[derive(Debug, Clone)]
pub struct KmsAuditRecord {
/// Correlates every record emitted for one logical request.
pub operation_id: Uuid,
/// The audited operation.
pub operation: KmsAuditOperation,
/// Event name published to audit consumers.
pub event: EventName,
/// Authenticated identity that requested the operation, or
/// [`OperationContext::INTERNAL_PRINCIPAL`] for server-initiated work.
pub principal: String,
/// Client address, when the caller supplied one.
pub source_ip: Option<String>,
/// Client user agent, when the caller supplied one.
pub user_agent: Option<String>,
/// Key the operation acted on. `None` for operations that span keys, such
/// as listing.
pub key_id: Option<String>,
/// Key version the operation resolved to, when the result identifies one.
/// Operations on a key as a whole leave this unset.
pub key_version: Option<u32>,
/// Whether the operation succeeded.
pub outcome: KmsAuditOutcome,
/// Failure category; `None` on success. See [`error_class`].
pub error_class: Option<&'static str>,
/// Backend that served the operation, e.g. `local` or `vault-transit`.
pub backend: &'static str,
/// Wall-clock time spent in the audited operation.
pub latency: Duration,
/// Retries performed above the audit point. `None` means the number is
/// not observable here: backend-internal retries are accounted for by the
/// `rustfs_kms_backend_operation_attempts` metric instead of being
/// duplicated (and possibly contradicted) in the audit trail.
pub retry_count: Option<u32>,
/// Server-supplied correlation values carried by the operation context.
/// Also the reserved slot for tenant attribution.
pub context: BTreeMap<String, String>,
/// Caller-supplied encryption context, after [`redact_encryption_context`].
pub encryption_context: BTreeMap<String, String>,
}
impl KmsAuditRecord {
/// Start a record for `operation` performed under `context`.
///
/// The outcome defaults to failure so that a record which somehow escapes
/// without [`Self::with_result`] under-reports success rather than
/// inventing it.
pub fn new(operation: KmsAuditOperation, context: &OperationContext, backend: &'static str) -> Self {
Self {
operation_id: context.operation_id,
operation,
event: operation.event_name(),
principal: context.principal.clone(),
source_ip: context.source_ip.clone(),
user_agent: context.user_agent.clone(),
key_id: None,
key_version: None,
outcome: KmsAuditOutcome::Failure,
error_class: None,
backend,
latency: Duration::ZERO,
retry_count: None,
context: context
.additional_context
.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect(),
encryption_context: BTreeMap::new(),
}
}
/// Attach the key the operation acted on.
pub fn with_key_id(mut self, key_id: Option<impl Into<String>>) -> Self {
self.key_id = key_id.map(Into::into);
self
}
/// Attach the key version the operation resolved to.
pub fn with_key_version(mut self, key_version: Option<u32>) -> Self {
self.key_version = key_version;
self
}
/// Attach the measured duration of the operation.
pub fn with_latency(mut self, latency: Duration) -> Self {
self.latency = latency;
self
}
/// Derive outcome and error class from the operation result.
pub fn with_result<T>(mut self, result: &crate::error::Result<T>) -> Self {
match result {
Ok(_) => {
self.outcome = KmsAuditOutcome::Success;
self.error_class = None;
}
Err(error) => {
self.outcome = KmsAuditOutcome::Failure;
self.error_class = Some(error_class(error));
}
}
self
}
/// Attach the caller-supplied encryption context, redacted.
///
/// Redaction happens here rather than at the call site so no caller can
/// place a raw context value into a record by accident.
pub fn with_encryption_context(mut self, encryption_context: &HashMap<String, String>) -> Self {
self.encryption_context = redact_encryption_context(encryption_context);
self
}
}
/// Reduce a caller-supplied encryption context to something safe to persist.
///
/// Keys naming the object's location are kept verbatim because they are the
/// reason the context is audited at all. Every other value is replaced by a
/// truncated digest, which still correlates repeated values across records
/// but does not reproduce whatever the caller put there.
pub fn redact_encryption_context(encryption_context: &HashMap<String, String>) -> BTreeMap<String, String> {
encryption_context
.iter()
.map(|(key, value)| {
let recorded = if ENCRYPTION_CONTEXT_ALLOWLIST.contains(&key.as_str()) {
value.clone()
} else {
digest_value(value)
};
(key.clone(), recorded)
})
.collect()
}
fn digest_value(value: &str) -> String {
let digest = hex::encode(Sha256::digest(value.as_bytes()));
format!("{DIGEST_PREFIX}{}", &digest[..DIGEST_LEN])
}
/// Receives KMS audit records.
///
/// Implementations must not block: they are called on the task that served
/// the KMS operation. Delivery failures are the sink's problem — the KMS
/// operation has already completed by the time a record is emitted, and its
/// result is never changed by what the sink does.
pub trait KmsAuditSink: Send + Sync {
/// Handle one audit record.
fn emit(&self, record: KmsAuditRecord);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn every_operation_maps_to_a_kms_event() {
let operations = [
KmsAuditOperation::CreateKey,
KmsAuditOperation::DescribeKey,
KmsAuditOperation::ListKeys,
KmsAuditOperation::ScheduleKeyDeletion,
KmsAuditOperation::CancelKeyDeletion,
KmsAuditOperation::EnableKey,
KmsAuditOperation::DisableKey,
KmsAuditOperation::RotateKey,
KmsAuditOperation::DeleteKey,
];
for operation in operations {
let event = operation.event_name();
assert!(event.is_kms(), "{} must map to a KMS event, got {event}", operation.as_str());
}
}
#[test]
fn record_defaults_to_failure_until_a_result_is_attached() {
let context = OperationContext::new("tester".to_string());
let record = KmsAuditRecord::new(KmsAuditOperation::CreateKey, &context, "local");
assert_eq!(record.outcome, KmsAuditOutcome::Failure);
assert!(record.error_class.is_none());
let ok: crate::error::Result<()> = Ok(());
assert_eq!(record.clone().with_result(&ok).outcome, KmsAuditOutcome::Success);
let denied: crate::error::Result<()> = Err(KmsError::access_denied("nope"));
let failed = record.with_result(&denied);
assert_eq!(failed.outcome, KmsAuditOutcome::Failure);
assert_eq!(failed.error_class, Some("access_denied"));
}
#[test]
fn record_carries_the_full_operation_context() {
let context = OperationContext::new("arn:aws:iam::user/alice".to_string())
.with_source_ip("10.0.0.7".to_string())
.with_user_agent("aws-cli/2".to_string())
.with_context("requestID".to_string(), "req-1".to_string());
let record = KmsAuditRecord::new(KmsAuditOperation::RotateKey, &context, "vault-transit")
.with_key_id(Some("key-1"))
.with_key_version(Some(3))
.with_latency(Duration::from_millis(12));
assert_eq!(record.operation_id, context.operation_id);
assert_eq!(record.principal, "arn:aws:iam::user/alice");
assert_eq!(record.source_ip.as_deref(), Some("10.0.0.7"));
assert_eq!(record.user_agent.as_deref(), Some("aws-cli/2"));
assert_eq!(record.key_id.as_deref(), Some("key-1"));
assert_eq!(record.key_version, Some(3));
assert_eq!(record.backend, "vault-transit");
assert_eq!(record.latency, Duration::from_millis(12));
assert_eq!(record.event, EventName::KmsKeyRotated);
assert_eq!(record.context.get("requestID").map(String::as_str), Some("req-1"));
}
#[test]
fn encryption_context_keeps_only_allowlisted_values_verbatim() {
let secret = "eyJhbGciOiJIUzI1NiJ9.super-secret-grant-token";
let context = HashMap::from([
("bucket".to_string(), "photos".to_string()),
("object_key".to_string(), "2026/cat.png".to_string()),
("algorithm".to_string(), "AES256".to_string()),
("grant_token".to_string(), secret.to_string()),
("customer_reference".to_string(), "acct-4711".to_string()),
]);
let redacted = redact_encryption_context(&context);
assert_eq!(redacted.get("bucket").map(String::as_str), Some("photos"));
assert_eq!(redacted.get("object_key").map(String::as_str), Some("2026/cat.png"));
assert_eq!(redacted.get("algorithm").map(String::as_str), Some("AES256"));
for key in ["grant_token", "customer_reference"] {
let value = redacted.get(key).expect("non-allowlisted key should be kept as a digest");
assert!(value.starts_with(DIGEST_PREFIX), "{key} should be digested, got {value}");
}
let rendered = format!("{redacted:?}");
assert!(!rendered.contains(secret), "redacted context must not reproduce the raw value");
assert!(!rendered.contains("acct-4711"), "redacted context must not reproduce the raw value");
}
#[test]
fn identical_values_digest_identically_across_records() {
// Correlation is the reason a digest is preferable to a fixed
// placeholder; assert it actually holds.
let first = redact_encryption_context(&HashMap::from([("tenant".to_string(), "alpha".to_string())]));
let second = redact_encryption_context(&HashMap::from([("tenant".to_string(), "alpha".to_string())]));
let other = redact_encryption_context(&HashMap::from([("tenant".to_string(), "beta".to_string())]));
assert_eq!(first.get("tenant"), second.get("tenant"));
assert_ne!(first.get("tenant"), other.get("tenant"));
}
}
File diff suppressed because it is too large Load Diff
+3 -75
View File
@@ -14,10 +14,7 @@
//! Local file-based KMS backend implementation
use crate::backends::{
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits,
ensure_tag_keys_are_mutable,
};
use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits};
use crate::config::KmsConfig;
use crate::config::LocalConfig;
use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
@@ -1297,32 +1294,6 @@ impl LocalKmsClient {
Ok(())
}
/// Read-modify-write of a key's mutable metadata under the per-key write
/// lock, carrying the existing key material over untouched.
///
/// `mutate` reports whether it changed anything; an unchanged record is
/// never rewritten, so a repeated no-op update neither rewrites the key
/// file nor risks a failed commit on an unrelated call.
async fn update_key_metadata<F>(&self, key_id: &str, mutate: F) -> Result<()>
where
F: FnOnce(&mut MasterKeyInfo) -> Result<bool>,
{
let _write_guard = self.lock_key_for_write(key_id).await;
let mut master_key = self.load_master_key(key_id).await?;
if !mutate(&mut master_key)? {
return Ok(());
}
// Preserve the existing key material (see enable_key): a metadata edit
// must never regenerate the master key, or every DEK wrapped by it
// becomes undecryptable.
let key_material = self.get_key_material(key_id).await?;
self.save_master_key(&master_key, &key_material).await?;
debug!(key_id, "Local KMS key metadata updated");
Ok(())
}
/// Test-only lifecycle driver: the product path goes through [`KmsBackend`].
#[cfg(test)]
pub(crate) async fn schedule_key_deletion(
@@ -1415,8 +1386,7 @@ impl LocalKmsBackend {
crate::config::BackendConfig::Local(local_config) => local_config.clone(),
crate::config::BackendConfig::VaultKv2(_)
| crate::config::BackendConfig::VaultTransit(_)
| crate::config::BackendConfig::Static(_)
| crate::config::BackendConfig::Aws(_) => {
| crate::config::BackendConfig::Static(_) => {
return Err(KmsError::configuration_error("Expected Local backend configuration"));
}
};
@@ -1441,15 +1411,12 @@ impl KmsBackend for LocalKmsBackend {
// Generate key material
let key_material = generate_key_material(algorithm)?;
let mut master_key = MasterKeyInfo::new_with_description(
let master_key = MasterKeyInfo::new_with_description(
key_id.clone(),
algorithm.to_string(),
Some("local-kms".to_string()),
request.description.clone(),
);
// Persist the caller's tags: the response below reports them, and
// describe_key reads them back out of this record.
master_key.metadata = request.tags.clone();
// Save to disk
self.client.save_new_master_key(&master_key, &key_material).await?;
@@ -1717,44 +1684,6 @@ impl KmsBackend for LocalKmsBackend {
self.client.disable_key(key_id, None).await
}
async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
self.client
.update_key_metadata(key_id, |master_key| {
if master_key.description.as_deref() == description {
return Ok(false);
}
master_key.description = description.map(str::to_string);
Ok(true)
})
.await
}
async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
self.client
.update_key_metadata(key_id, |master_key| {
let mut changed = false;
for (tag_key, value) in tags {
changed |= master_key.metadata.insert(tag_key.clone(), value.clone()).as_ref() != Some(value);
}
Ok(changed)
})
.await
}
async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
self.client
.update_key_metadata(key_id, |master_key| {
let mut changed = false;
for tag_key in tag_keys {
changed |= master_key.metadata.remove(tag_key).is_some();
}
Ok(changed)
})
.await
}
async fn health_check(&self) -> Result<bool> {
self.client.health_check().await.map(|_| true)
}
@@ -1767,7 +1696,6 @@ impl KmsBackend for LocalKmsBackend {
.with_enable_disable(true)
.with_schedule_deletion(true)
.with_physical_delete(true)
.with_update_key_metadata(true)
}
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
-90
View File
@@ -19,9 +19,7 @@ use crate::types::*;
use async_trait::async_trait;
use jiff::Zoned;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
pub mod aws;
#[cfg(test)]
mod contract_tests;
pub mod local;
@@ -100,32 +98,6 @@ pub(crate) fn ensure_key_status_permits(key_id: &str, status: &KeyStatus, operat
ensure_key_state_permits(key_id, &state, operation)
}
/// Tag key that carries the key's identity rather than user metadata.
///
/// The key-creation path lifts `name` out of the caller's tag map and uses it
/// as the key id, so a key's `name` tag and its id are the same string.
/// Rewriting or dropping it after creation would leave the key addressable
/// under an id its own metadata no longer states. Metadata updates therefore
/// reject it; only creation may set it.
pub const RESERVED_KEY_NAME_TAG: &str = "name";
/// Reject a metadata update that would rewrite or remove
/// [`RESERVED_KEY_NAME_TAG`].
///
/// Enforced by every backend that implements tag updates, so no call path —
/// including a direct [`KmsBackend`] user that bypasses the manager — can
/// detach a key from its identity.
pub(crate) fn ensure_tag_keys_are_mutable<'a>(tag_keys: impl IntoIterator<Item = &'a str>) -> Result<()> {
for tag_key in tag_keys {
if tag_key == RESERVED_KEY_NAME_TAG {
return Err(KmsError::invalid_parameter(format!(
"Tag '{RESERVED_KEY_NAME_TAG}' identifies the key and cannot be updated or removed"
)));
}
}
Ok(())
}
/// Simplified KMS backend interface for manager
#[async_trait]
pub trait KmsBackend: Send + Sync {
@@ -181,38 +153,6 @@ pub trait KmsBackend: Send + Sync {
Err(KmsError::unsupported_capability("backend without rotation support", "rotate_key"))
}
/// Replace a key's free-form description; `None` clears it.
///
/// Backends that advertise [`BackendCapabilities::update_key_metadata`]
/// must override this method; the default rejects the operation.
async fn update_key_description(&self, _key_id: &str, _description: Option<&str>) -> Result<()> {
Err(KmsError::unsupported_capability(
"backend without key metadata updates",
"update_key_description",
))
}
/// Add or overwrite the given tags, leaving every other tag untouched.
///
/// Implementations must run [`ensure_tag_keys_are_mutable`] before
/// persisting anything. Backends that advertise
/// [`BackendCapabilities::update_key_metadata`] must override this method;
/// the default rejects the operation.
async fn tag_key(&self, _key_id: &str, _tags: &HashMap<String, String>) -> Result<()> {
Err(KmsError::unsupported_capability("backend without key metadata updates", "tag_key"))
}
/// Remove the given tags.
///
/// Tags that are not set are ignored, so repeating the call is a no-op
/// rather than an error. Implementations must run
/// [`ensure_tag_keys_are_mutable`] before persisting anything. Backends
/// that advertise [`BackendCapabilities::update_key_metadata`] must
/// override this method; the default rejects the operation.
async fn untag_key(&self, _key_id: &str, _tag_keys: &[String]) -> Result<()> {
Err(KmsError::unsupported_capability("backend without key metadata updates", "untag_key"))
}
/// Health check
async fn health_check(&self) -> Result<bool>;
@@ -282,8 +222,6 @@ pub struct BackendCapabilities {
pub versioning: bool,
/// Irreversible physical deletion of key material
pub physical_delete: bool,
/// Updating a key's description and tags after creation
pub update_key_metadata: bool,
}
impl BackendCapabilities {
@@ -300,7 +238,6 @@ impl BackendCapabilities {
schedule_deletion: false,
versioning: false,
physical_delete: false,
update_key_metadata: false,
}
}
@@ -351,12 +288,6 @@ impl BackendCapabilities {
self.physical_delete = physical_delete;
self
}
/// Set whether description and tag updates are supported
pub const fn with_update_key_metadata(mut self, update_key_metadata: bool) -> Self {
self.update_key_metadata = update_key_metadata;
self
}
}
impl Default for BackendCapabilities {
@@ -435,7 +366,6 @@ mod tests {
assert!(!capabilities.schedule_deletion);
assert!(!capabilities.versioning);
assert!(!capabilities.physical_delete);
assert!(!capabilities.update_key_metadata);
}
#[tokio::test]
@@ -444,12 +374,6 @@ mod tests {
("enable_key", MinimalBackend.enable_key("any-key").await),
("disable_key", MinimalBackend.disable_key("any-key").await),
("rotate_key", MinimalBackend.rotate_key("any-key").await),
(
"update_key_description",
MinimalBackend.update_key_description("any-key", Some("new")).await,
),
("tag_key", MinimalBackend.tag_key("any-key", &HashMap::new()).await),
("untag_key", MinimalBackend.untag_key("any-key", &[]).await),
] {
let error = result.expect_err("backends must opt in to lifecycle operations by overriding them");
assert!(
@@ -471,20 +395,6 @@ mod tests {
);
}
#[test]
fn identity_tag_is_rejected_by_metadata_updates() {
let error = ensure_tag_keys_are_mutable([RESERVED_KEY_NAME_TAG])
.expect_err("the identity tag must not be writable through a metadata update");
assert!(
matches!(&error, KmsError::InvalidOperation { message } if message.contains(RESERVED_KEY_NAME_TAG)),
"expected a typed rejection naming the tag, got {error:?}"
);
// Ordinary tags — including ones that merely contain the reserved name
// — stay writable.
ensure_tag_keys_are_mutable(["team", "nickname", "Name"]).expect("ordinary tags must remain writable");
}
#[tokio::test]
async fn local_backend_capabilities_golden() {
let temp_dir = tempfile::tempdir().expect("temp dir should be created");
@@ -1,15 +0,0 @@
---
source: crates/kms/src/backends/aws.rs
expression: capabilities_snapshot(backend.capabilities())
---
{
"decrypt": true,
"enable_disable": true,
"encrypt": true,
"generate_data_key": true,
"physical_delete": false,
"rotate": true,
"schedule_deletion": true,
"update_key_metadata": false,
"versioning": true
}
@@ -10,6 +10,5 @@ expression: capabilities_snapshot(backend.capabilities())
"physical_delete": true,
"rotate": false,
"schedule_deletion": true,
"update_key_metadata": true,
"versioning": false
}
@@ -10,6 +10,5 @@ expression: capabilities_snapshot(backend.capabilities())
"physical_delete": false,
"rotate": false,
"schedule_deletion": false,
"update_key_metadata": false,
"versioning": false
}
@@ -10,6 +10,5 @@ expression: capabilities_snapshot(backend.capabilities())
"physical_delete": true,
"rotate": true,
"schedule_deletion": true,
"update_key_metadata": true,
"versioning": true
}
@@ -10,6 +10,5 @@ expression: capabilities_snapshot(backend.capabilities())
"physical_delete": true,
"rotate": true,
"schedule_deletion": true,
"update_key_metadata": true,
"versioning": true
}
-26
View File
@@ -431,32 +431,6 @@ mod tests {
(backend, key_id, key)
}
#[tokio::test]
async fn metadata_updates_report_the_capability_gap() {
let (backend, key_id, _key) = create_test_backend().await;
assert!(
!backend.capabilities().update_key_metadata,
"Static KMS owns no mutable key record, so it must not advertise metadata updates"
);
for (operation, result) in [
("update_key_description", backend.update_key_description(&key_id, Some("new")).await),
(
"tag_key",
backend
.tag_key(&key_id, &HashMap::from([("team".to_string(), "storage".to_string())]))
.await,
),
("untag_key", backend.untag_key(&key_id, &["team".to_string()]).await),
] {
let error = result.expect_err("Static KMS is read-only: metadata updates must be rejected");
assert!(
matches!(error, KmsError::UnsupportedCapability { .. }),
"expected UnsupportedCapability for {operation}, got {error:?}"
);
}
}
#[tokio::test]
async fn test_generate_and_decrypt_data_key() {
let (backend, key_id, _key) = create_test_backend().await;
+1 -134
View File
@@ -20,7 +20,6 @@ use crate::backends::vault_credentials::{
};
use crate::backends::{
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits, ensure_key_status_permits,
ensure_tag_keys_are_mutable,
};
use crate::config::{KmsConfig, VaultConfig};
use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
@@ -1021,67 +1020,6 @@ impl VaultKmsClient {
Ok(())
}
/// Replace the key's description; `None` clears it.
///
/// The write goes through the check-and-set read-modify-write loop, so a
/// rotation or state transition landing in between is carried over instead
/// of clobbered. A description that already matches is not rewritten.
pub(crate) async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
self.update_key_data_with_cas(key_id, |key_data| {
if key_data.description.as_deref() == description {
return Ok(CasMutation::Skip(()));
}
key_data.description = description.map(str::to_string);
Ok(CasMutation::Write(()))
})
.await?;
debug!(key_id, "Vault KMS key description updated");
Ok(())
}
/// Add or overwrite tags, leaving every other tag untouched.
pub(crate) async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
self.update_key_data_with_cas(key_id, |key_data| {
let mut changed = false;
for (tag_key, value) in tags {
changed |= key_data.tags.insert(tag_key.clone(), value.clone()).as_ref() != Some(value);
}
Ok(if changed {
CasMutation::Write(())
} else {
CasMutation::Skip(())
})
})
.await?;
debug!(key_id, "Vault KMS key tags updated");
Ok(())
}
/// Remove tags; tags that are not set are ignored.
pub(crate) async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
self.update_key_data_with_cas(key_id, |key_data| {
let mut changed = false;
for tag_key in tag_keys {
changed |= key_data.tags.remove(tag_key).is_some();
}
Ok(if changed {
CasMutation::Write(())
} else {
CasMutation::Skip(())
})
})
.await?;
debug!(key_id, "Vault KMS key tags removed");
Ok(())
}
/// Rotate the master key while keeping every historical version decryptable.
///
/// Commit protocol (all writes check-and-set, in this order):
@@ -1225,8 +1163,7 @@ impl VaultKmsBackend {
crate::config::BackendConfig::VaultKv2(vault_config) => (**vault_config).clone(),
crate::config::BackendConfig::Local(_)
| crate::config::BackendConfig::VaultTransit(_)
| crate::config::BackendConfig::Static(_)
| crate::config::BackendConfig::Aws(_) => {
| crate::config::BackendConfig::Static(_) => {
return Err(KmsError::configuration_error("Expected Vault KV2 backend configuration"));
}
};
@@ -1509,18 +1446,6 @@ impl KmsBackend for VaultKmsBackend {
self.client.rotate_key(key_id, None).await.map(|_| ())
}
async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
self.client.update_key_description(key_id, description).await
}
async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
self.client.tag_key(key_id, tags).await
}
async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
self.client.untag_key(key_id, tag_keys).await
}
async fn health_check(&self) -> Result<bool> {
self.client.health_check().await.map(|_| true)
}
@@ -1536,7 +1461,6 @@ impl KmsBackend for VaultKmsBackend {
.with_schedule_deletion(true)
.with_versioning(true)
.with_physical_delete(true)
.with_update_key_metadata(true)
}
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
@@ -2920,63 +2844,6 @@ mod tests {
);
}
/// Tag updates are check-and-set read-modify-writes over the live record,
/// never blind overwrites: they preserve the material and the tags they did
/// not address.
#[tokio::test]
async fn wired_tag_key_writeback_is_check_and_set() {
let mut key_data = healthy_key_data();
key_data.tags = HashMap::from([("name".to_string(), "wired-key".to_string())]);
let (vault, client) = scripted_client(vec![
ScriptedResponse::ok(kv2_metadata_read_data(1)),
ScriptedResponse::ok(kv2_read_data(&key_data)),
ScriptedResponse::ok(kv2_write_ack()),
])
.await;
client
.tag_key("wired-key", &HashMap::from([("team".to_string(), "storage".to_string())]))
.await
.expect("tagging must succeed");
let bodies = vault.request_bodies();
let writeback = parse_write_body(&bodies[2]);
assert_eq!(writeback["options"]["cas"], serde_json::json!(1), "{writeback}");
assert_eq!(writeback["data"]["tags"]["team"], serde_json::json!("storage"), "{writeback}");
assert_eq!(
writeback["data"]["tags"]["name"],
serde_json::json!("wired-key"),
"a tag update must not drop tags it did not address: {writeback}"
);
assert_eq!(
writeback["data"]["encrypted_key_material"],
serde_json::json!(healthy_key_data().encrypted_key_material),
"the write-back must preserve the material of the freshly read record: {writeback}"
);
}
/// Rejecting the identity tag happens before any Vault call, so a rejected
/// request cannot leave a partial write behind.
#[tokio::test]
async fn wired_identity_tag_update_is_rejected_before_any_vault_call() {
let (vault, client) = scripted_client(Vec::new()).await;
for result in [
client
.tag_key("wired-key", &HashMap::from([("name".to_string(), "other".to_string())]))
.await,
client.untag_key("wired-key", &["name".to_string()]).await,
] {
let error = result.expect_err("the identity tag must not be writable");
assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}");
}
assert!(
vault.request_bodies().is_empty(),
"a rejected metadata update must not reach Vault: {:?}",
vault.request_bodies()
);
}
/// A version record above the current pointer means the top-level record
/// regressed (a lost update rolled back a committed rotation). Resolving
/// material through such a record must fail closed instead of quietly
+8 -208
View File
@@ -57,42 +57,6 @@ const DEFAULT_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(5);
/// Default seconds between token file re-reads for [`TokenFileSource`].
const DEFAULT_TOKEN_FILE_POLL_INTERVAL_SECS: u64 = 30;
// ---------------------------------------------------------------------------
// Metrics
//
// Both gauges describe the one credential generation currently installed, so
// they carry no labels: the Vault address, mount, auth path and token are all
// off limits as label values, and there is exactly one generation to describe.
// The renewal loop republishes them on a bounded cadence while it waits, so a
// scrape landing between refresh cycles never reads a TTL frozen at the last
// refresh, or a fail-closed state that flipped after it.
// ---------------------------------------------------------------------------
/// Gauge: seconds left before the active Vault token expires; `0` once it has.
const METRIC_TOKEN_TTL_SECONDS: &str = "rustfs_kms_vault_token_ttl_seconds";
/// Gauge: `1` while [`VaultCredentialProvider::current`] refuses to hand out
/// the token because it is inside the fail-closed safety window, `0` otherwise.
const METRIC_CREDENTIALS_FAIL_CLOSED: &str = "rustfs_kms_vault_credentials_fail_closed";
/// How often the renewal loop republishes the credential gauges while waiting.
/// Bounds how stale a scrape can be, without any additional Vault traffic.
const CREDENTIAL_GAUGE_INTERVAL: Duration = Duration::from_secs(10);
/// Register metric descriptions once per process.
fn describe_credential_metrics() {
static DESCRIBE: std::sync::Once = std::sync::Once::new();
DESCRIBE.call_once(|| {
metrics::describe_gauge!(
METRIC_TOKEN_TTL_SECONDS,
"Seconds remaining before the Vault token backing the KMS backend expires"
);
metrics::describe_gauge!(
METRIC_CREDENTIALS_FAIL_CLOSED,
"1 while the Vault credential provider refuses to serve its token because the token is inside the fail-closed safety window"
);
});
}
/// A crate-owned secret value, zeroized on drop and redacted in Debug output.
#[derive(Clone, Zeroize, ZeroizeOnDrop)]
pub(crate) struct SecretString(String);
@@ -663,45 +627,6 @@ impl VaultCredentialProvider {
Ok(handle)
}
/// Publish the credential gauges for the generation currently installed.
///
/// The fail-closed gauge re-evaluates the very gate
/// [`VaultCredentialProvider::current`] applies, so what operators see and
/// what the request path does cannot drift apart.
fn record_credential_gauges(&self) {
let handle = self.current.load();
let now = Instant::now();
let fail_closed = match handle.expires_at() {
Some(expires_at) => {
metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
now + self.policy.safety_window >= expires_at
}
// A generation without an expiry has no remaining TTL to report
// and can never lapse, so it can never fail closed either.
None => false,
};
metrics::gauge!(METRIC_CREDENTIALS_FAIL_CLOSED).set(if fail_closed { 1.0 } else { 0.0 });
}
/// Wait until `deadline`, republishing the credential gauges on the
/// observation cadence. Reports `false` when cancellation cut the wait
/// short.
async fn wait_publishing_gauges(&self, deadline: Instant, cancel: &CancellationToken) -> bool {
loop {
self.record_credential_gauges();
let now = Instant::now();
if now >= deadline {
return true;
}
let slice = (deadline - now).min(CREDENTIAL_GAUGE_INTERVAL);
tokio::select! {
biased;
_ = cancel.cancelled() => return false,
_ = tokio::time::sleep(slice) => {}
}
}
}
/// Refresh the credentials if generation `observed` is still current.
///
/// Single-flight: concurrent callers serialize on the refresh lock, and a
@@ -786,22 +711,17 @@ impl fmt::Debug for VaultCredentialProvider {
/// again immediately (the renewal point is already in the past), so the
/// provider keeps trying to recover even after the fail-closed window has
/// been reached.
///
/// Both waits run through [`VaultCredentialProvider::wait_publishing_gauges`],
/// which is the only place the credential gauges are published: the loop is
/// already the component that tracks token expiry, and doing it here keeps the
/// request path free of any metric work.
async fn renewal_loop(provider: Arc<VaultCredentialProvider>, cancel: CancellationToken) {
describe_credential_metrics();
loop {
let handle = provider.snapshot();
let Some(renew_at) = handle.renew_at() else {
// The current generation never expires; nothing left to schedule.
provider.record_credential_gauges();
return;
};
if !provider.wait_publishing_gauges(renew_at, &cancel).await {
return;
tokio::select! {
biased;
_ = cancel.cancelled() => return,
_ = tokio::time::sleep_until(renew_at) => {}
}
match provider.refresh(handle.generation, &cancel).await {
@@ -813,9 +733,10 @@ async fn renewal_loop(provider: Arc<VaultCredentialProvider>, cancel: Cancellati
error = %error,
"Vault credential refresh failed; retrying until the credentials recover"
);
let retry_at = Instant::now() + provider.policy.retry_interval;
if !provider.wait_publishing_gauges(retry_at, &cancel).await {
return;
tokio::select! {
biased;
_ = cancel.cancelled() => return,
_ = tokio::time::sleep(provider.policy.retry_interval) => {}
}
}
}
@@ -1433,125 +1354,4 @@ mod tests {
assert!(!rendered.contains(TEST_TOKEN), "debug output must not leak the vault token: {rendered}");
assert!(rendered.contains("vault-token"), "the file path is not a secret");
}
// -- Metric emission ----------------------------------------------------
//
// Each test installs a thread-local debugging recorder and drives a
// paused-clock current-thread runtime inside it, so the renewal loop's
// gauge publications land on exact virtual timestamps.
use metrics_util::MetricKind;
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
type MetricEntry = (
metrics_util::CompositeKey,
Option<metrics::Unit>,
Option<metrics::SharedString>,
DebugValue,
);
/// Run `test` on a paused current-thread runtime under a debugging
/// recorder and return one snapshot of everything it emitted.
///
/// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
/// the recorded state, so taking it per assertion would only show the
/// first assertion any data.
fn record_metrics<Out>(
test: impl FnOnce() -> std::pin::Pin<Box<dyn std::future::Future<Output = Out>>>,
) -> (Vec<MetricEntry>, Out) {
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let out = metrics::with_local_recorder(&recorder, || {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_time()
.start_paused(true)
.build()
.expect("current-thread runtime must build");
runtime.block_on(test())
});
(snapshotter.snapshot().into_vec(), out)
}
/// Last value of a gauge, plus the labels it was published with.
fn gauge(snapshot: &[MetricEntry], name: &str) -> Option<(f64, Vec<String>)> {
snapshot.iter().find_map(|(composite, _unit, _description, value)| {
let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
match (matches, value) {
(true, DebugValue::Gauge(value)) => Some((
value.into_inner(),
composite.key().labels().map(|label| label.key().to_string()).collect(),
)),
_ => None,
}
})
}
#[test]
fn renewal_loop_republishes_token_ttl_while_it_waits() {
let (snapshot, ()) = record_metrics(|| {
Box::pin(async {
let (provider, _state) = scripted_provider(
Duration::from_secs(60),
true,
test_policy(Duration::from_secs(10), Duration::from_secs(5)),
)
.await;
let task = provider.spawn_renewal_task().expect("lease-bound tokens need renewal");
// Well inside the first half of the lease: nothing has been
// renewed yet, so only the observation cadence can have moved
// the gauge off its initial 60s.
tokio::time::sleep(Duration::from_secs(25)).await;
task.shutdown().await;
})
});
let (ttl, labels) = gauge(&snapshot, METRIC_TOKEN_TTL_SECONDS).expect("token TTL gauge must be published");
assert!(
(ttl - 40.0).abs() < 1.0,
"expected ~40s left of a 60s lease at the last observation, got {ttl}"
);
assert!(labels.is_empty(), "credential gauges must carry no labels, got {labels:?}");
assert_eq!(
gauge(&snapshot, METRIC_CREDENTIALS_FAIL_CLOSED).map(|(value, _)| value),
Some(0.0),
"a token outside its safety window must not report fail-closed"
);
}
#[test]
fn fail_closed_gauge_tracks_the_gate_the_request_path_applies() {
let (snapshot, refused) = record_metrics(|| {
Box::pin(async {
let (provider, state) = scripted_provider(
Duration::from_secs(60),
true,
test_policy(Duration::from_secs(10), Duration::from_secs(5)),
)
.await;
state.fail_renew.store(true, Ordering::SeqCst);
state.fail_login.store(true, Ordering::SeqCst);
let task = provider.spawn_renewal_task().expect("renewal task");
// 60s lease minus the 10s safety window: by 51s the provider
// is refusing the token, and the gauge must already say so.
tokio::time::sleep(Duration::from_secs(51)).await;
let refused = provider.current().is_err();
task.shutdown().await;
refused
})
});
assert!(refused, "a token inside the safety window must be refused");
assert_eq!(
gauge(&snapshot, METRIC_CREDENTIALS_FAIL_CLOSED).map(|(value, _)| value),
Some(1.0),
"the gauge must report the same fail-closed state the request path enforces"
);
let (ttl, _labels) = gauge(&snapshot, METRIC_TOKEN_TTL_SECONDS).expect("token TTL gauge must be published");
assert!(
(ttl - 10.0).abs() < 1.0,
"expected the TTL gauge to track the lease down into its safety window, got {ttl}"
);
}
}
+2 -60
View File
@@ -18,10 +18,7 @@ use crate::backends::vault_credentials::{
CredentialTaskHandle, VaultClientHandle, VaultConnectionSettings, VaultCredentialPolicy, VaultCredentialProvider,
token_source_for,
};
use crate::backends::{
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits,
ensure_tag_keys_are_mutable,
};
use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits};
use crate::config::{KmsConfig, VaultTransitConfig};
use crate::encryption::{DataKeyEnvelope, generate_key_material};
use crate::error::{KmsError, Result};
@@ -911,46 +908,6 @@ impl VaultTransitKmsClient {
.map(|_| ())
}
/// Replace the key's description; `None` clears it.
///
/// Metadata edits carry no state gate: they neither use nor invalidate key
/// material, so they stay available for whatever lifecycle state the key
/// is in.
pub(crate) async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
self.mutate_key_metadata(key_id, |metadata| {
metadata.description = description.map(str::to_string);
Ok(())
})
.await
.map(|_| ())
}
/// Add or overwrite tags, leaving every other tag untouched.
pub(crate) async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
self.mutate_key_metadata(key_id, |metadata| {
metadata
.tags
.extend(tags.iter().map(|(key, value)| (key.clone(), value.clone())));
Ok(())
})
.await
.map(|_| ())
}
/// Remove tags; tags that are not set are ignored.
pub(crate) async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
self.mutate_key_metadata(key_id, |metadata| {
for tag_key in tag_keys {
metadata.tags.remove(tag_key);
}
Ok(())
})
.await
.map(|_| ())
}
/// Test-only lifecycle driver: the product path goes through [`KmsBackend`].
#[cfg(test)]
pub(crate) async fn schedule_key_deletion(
@@ -1055,9 +1012,7 @@ impl VaultTransitKmsBackend {
metadata_key_prefix: vault_config.key_path_prefix.clone(),
tls: vault_config.tls.clone(),
},
crate::config::BackendConfig::Local(_)
| crate::config::BackendConfig::Static(_)
| crate::config::BackendConfig::Aws(_) => {
crate::config::BackendConfig::Local(_) | crate::config::BackendConfig::Static(_) => {
return Err(KmsError::configuration_error("Expected Vault Transit backend configuration"));
}
};
@@ -1280,18 +1235,6 @@ impl KmsBackend for VaultTransitKmsBackend {
self.client.rotate_key(key_id, None).await.map(|_| ())
}
async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
self.client.update_key_description(key_id, description).await
}
async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
self.client.tag_key(key_id, tags).await
}
async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
self.client.untag_key(key_id, tag_keys).await
}
async fn health_check(&self) -> Result<bool> {
self.client.health_check().await.map(|_| true)
}
@@ -1306,7 +1249,6 @@ impl KmsBackend for VaultTransitKmsBackend {
.with_schedule_deletion(true)
.with_versioning(true)
.with_physical_delete(true)
.with_update_key_metadata(true)
}
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
+16 -7
View File
@@ -74,7 +74,7 @@ pub const LOCAL_BUNDLE_MANIFEST_FILE: &str = "manifest.json";
const ARTIFACTS_DIR: &str = "artifacts";
const KEYS_DIR: &str = "artifacts/keys";
const SALT_ARTIFACT_PATH: &str = "artifacts/master-key.salt.enc";
const AEAD_NONCE_LEN: usize = 12;
pub(crate) const AEAD_NONCE_LEN: usize = 12;
/// Domain-separation context for the artifact AAD binding.
const BUNDLE_AAD_CONTEXT: &str = "rustfs-kms-local-backup:v1";
/// Domain-separation context for the master-key verifier.
@@ -118,7 +118,7 @@ impl BackupKek {
}
}
fn cipher(&self) -> Aes256Gcm {
pub(crate) fn cipher(&self) -> Aes256Gcm {
Aes256Gcm::new(&Key::<Aes256Gcm>::from(*self.key))
}
}
@@ -227,11 +227,14 @@ pub async fn export_local_backup(
Ok(manifest)
}
/// Read and fully validate the manifest of a local bundle directory.
/// Read and fully validate the manifest of a bundle directory, whatever
/// backend produced it.
///
/// A directory without a manifest is an interrupted export: the manifest is
/// written last, so its absence means the bundle never sealed.
pub async fn read_local_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> {
/// written last, so its absence means the bundle never sealed. The manifest
/// file name and framing are bundle-wide, not Local-specific, so consumers of
/// other backends' bundles read them through here too.
pub async fn read_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> {
let manifest_path = bundle_dir.join(LOCAL_BUNDLE_MANIFEST_FILE);
let bytes = match fs::read(&manifest_path).await {
Ok(bytes) => bytes,
@@ -240,7 +243,12 @@ pub async fn read_local_bundle_manifest(bundle_dir: &Path) -> Result<BackupManif
}
Err(error) => return Err(error.into()),
};
let manifest = BackupManifest::decode(&bytes)?;
Ok(BackupManifest::decode(&bytes)?)
}
/// Read and fully validate the manifest of a *local* bundle directory.
pub async fn read_local_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> {
let manifest = read_bundle_manifest(bundle_dir).await?;
if manifest.backend != BackupBackendKind::Local {
return Err(
BackupError::corrupted(format!("bundle manifest declares backend {:?}, expected Local", manifest.backend)).into(),
@@ -424,6 +432,7 @@ async fn build_and_write_bundle(
backup_kek: kek.descriptor(),
artifacts,
local_kdf: Some(local_kdf_descriptor(snapshot, master_key_verifier)),
external_references: None,
key_versions: None,
capability_discovery: None,
completeness: CompletenessState::InProgress,
@@ -502,7 +511,7 @@ async fn encrypt_and_write_artifact(
/// AAD binding an artifact to its bundle identity and path. A JSON tuple
/// gives unambiguous field boundaries without a hand-rolled framing format.
fn artifact_aad(backup_id: &str, snapshot_generation: u64, artifact_path: &str) -> Vec<u8> {
pub(crate) fn artifact_aad(backup_id: &str, snapshot_generation: u64, artifact_path: &str) -> Vec<u8> {
serde_json::to_vec(&(BUNDLE_AAD_CONTEXT, backup_id, snapshot_generation, artifact_path))
.expect("AAD tuple of strings and integers always serializes")
}
+311
View File
@@ -306,6 +306,141 @@ impl LocalKdfDescriptor {
}
}
/// Transit engine reference recorded in a Vault-backed bundle.
///
/// Transit keys are non-exportable, so a bundle can only ever name the key and
/// the version window its content depends on. Restore compares these values
/// against the Transit key the operator's native Vault restore produced.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct VaultTransitReference {
/// Transit engine mount path.
pub mount_path: String,
/// Transit key name.
pub key_name: String,
/// Lowest Transit key version the bundle's content still needs. A target
/// whose `min_decryption_version` sits above this can no longer decrypt
/// the oldest state in the bundle.
pub required_min_version: u32,
/// Latest Transit key version at snapshot time. A target whose newest
/// version is below this was restored to a point before the bundle.
pub current_version: u32,
/// Operator-recorded immutable reference to the native Vault/HSM snapshot
/// protecting this key. RustFS neither produces nor consumes that
/// snapshot; the reference exists so restore evidence can name it.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub native_snapshot_reference: Option<String>,
}
/// KV generation of one key's record at snapshot time.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct VaultKvRecordReference {
/// Stable key id the record belongs to.
pub key_id: String,
/// KV2 secret version holding the record when the snapshot was taken.
/// Restore compares the target's current version against it: a lower
/// value means the target's Vault was restored to a point before the
/// bundle, a higher value means the target moved ahead of it.
pub kv_version: u64,
}
/// Immutable references to the external Vault state a bundle depends on.
///
/// A Vault-backed bundle never carries the cryptographic root: Transit keys
/// cannot be exported and KV2 records live in Vault's own storage. What the
/// bundle can own is a precise description of *which* external state it was
/// captured against, so a restore refuses to proceed when the operator's
/// native Vault restore landed somewhere else. Credentials are structurally
/// absent — no token, AppRole secret id, or TLS material is representable in
/// this type.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct VaultExternalReferences {
/// Opaque identity of the Vault cluster the snapshot was taken from.
pub cluster_id: String,
/// Vault Enterprise namespace, when the deployment uses one.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub namespace: Option<String>,
/// KV v2 mount holding the RustFS records.
pub kv_mount: String,
/// Path prefix under `kv_mount` holding the RustFS records.
pub kv_path_prefix: String,
/// Transit engine reference; present exactly when the bundle's material
/// is protected by a Transit key.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub transit: Option<VaultTransitReference>,
/// Per-key KV generation observed at snapshot time, one entry per key the
/// bundle carries a record for.
pub kv_records: Vec<VaultKvRecordReference>,
}
impl VaultExternalReferences {
fn validate(&self, backend: BackupBackendKind, protection: AtRestProtection) -> Result<(), BackupError> {
require_non_empty("external_references.cluster_id", &self.cluster_id)?;
require_non_empty("external_references.kv_mount", &self.kv_mount)?;
require_non_empty("external_references.kv_path_prefix", &self.kv_path_prefix)?;
if self.namespace.as_deref() == Some("") {
return Err(BackupError::corrupted("external Vault namespace must not be empty when present"));
}
// The Transit reference is required exactly where the cryptographic
// root lives in Transit; a storage-only KV2 bundle that claims one
// would misdescribe its own trust root.
let transit_required = matches!(
(backend, protection),
(BackupBackendKind::VaultTransit, _) | (BackupBackendKind::VaultKv2, AtRestProtection::TransitWrapped)
);
match (&self.transit, transit_required) {
(None, true) => {
return Err(BackupError::corrupted(format!(
"({backend:?}, {protection:?}) bundles must reference the Transit key protecting them"
)));
}
(Some(_), false) => {
return Err(BackupError::corrupted(format!(
"({backend:?}, {protection:?}) bundles have no Transit trust root to reference"
)));
}
(Some(transit), true) => {
require_non_empty("external_references.transit.mount_path", &transit.mount_path)?;
require_non_empty("external_references.transit.key_name", &transit.key_name)?;
if transit.required_min_version == 0 {
return Err(BackupError::corrupted("Transit reference must require at least key version 1"));
}
if transit.current_version < transit.required_min_version {
return Err(BackupError::corrupted(format!(
"Transit reference declares current version {} below the required minimum {}",
transit.current_version, transit.required_min_version
)));
}
if transit.native_snapshot_reference.as_deref() == Some("") {
return Err(BackupError::corrupted("Transit native snapshot reference must not be empty when present"));
}
}
(None, false) => {}
}
let mut seen = BTreeSet::new();
for record in &self.kv_records {
require_non_empty("external_references.kv_records[].key_id", &record.key_id)?;
if record.kv_version == 0 {
return Err(BackupError::corrupted(format!(
"KV generation for key '{}' must be at least 1",
record.key_id
)));
}
if !seen.insert(record.key_id.as_str()) {
return Err(BackupError::corrupted(format!(
"KV reference for key '{}' appears more than once",
record.key_id
)));
}
}
Ok(())
}
}
/// Completeness marker of a bundle.
///
/// A producer writes `in-progress` state (or no manifest at all) until the
@@ -394,6 +529,10 @@ pub struct BackupManifest {
/// Local backend section; required exactly when `backend` is `Local`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub local_kdf: Option<LocalKdfDescriptor>,
/// External Vault references; required exactly when `backend` is a Vault
/// backend. Never carries credentials — see [`VaultExternalReferences`].
#[serde(default, skip_serializing_if = "Option::is_none")]
pub external_references: Option<VaultExternalReferences>,
/// Reserved for the per-key version/envelope inventory defined by the
/// backlog#1565 contract. May not carry data in format version 1.
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -572,6 +711,7 @@ impl BackupManifest {
}
self.validate_responsibility()?;
self.validate_local_kdf()?;
self.validate_external_references()?;
self.validate_artifacts()?;
Ok(())
}
@@ -601,6 +741,19 @@ impl BackupManifest {
}
}
fn validate_external_references(&self) -> Result<(), BackupError> {
let vault_backend = matches!(self.backend, BackupBackendKind::VaultKv2 | BackupBackendKind::VaultTransit);
match (&self.external_references, vault_backend) {
(None, true) => Err(BackupError::corrupted("Vault backend manifest must carry external Vault references")),
(Some(_), false) => Err(BackupError::corrupted(format!(
"external Vault references are not valid for backend {:?}",
self.backend
))),
(Some(references), true) => references.validate(self.backend, self.at_rest_protection),
(None, false) => Ok(()),
}
}
fn validate_artifacts(&self) -> Result<(), BackupError> {
let mut paths = BTreeSet::new();
for artifact in &self.artifacts {
@@ -865,6 +1018,7 @@ mod tests {
vec![AtRestProtection::EncryptedMasterKey],
Some("verifier-opaque-1".to_string()),
)),
external_references: None,
key_versions: None,
capability_discovery: None,
completeness: CompletenessState::InProgress,
@@ -1271,4 +1425,161 @@ mod tests {
assert_eq!(decoded, sealed);
assert_eq!(decoded.responsibility, BackupResponsibility::ReferenceOnly);
}
fn transit_references() -> VaultExternalReferences {
VaultExternalReferences {
cluster_id: "vault-cluster-a".to_string(),
namespace: Some("tenant-a".to_string()),
kv_mount: "secret".to_string(),
kv_path_prefix: "rustfs/kms/transit-metadata".to_string(),
transit: Some(VaultTransitReference {
mount_path: "transit".to_string(),
key_name: "rustfs-master".to_string(),
required_min_version: 2,
current_version: 5,
native_snapshot_reference: Some("s3://dr/vault/2026-08-01.snap".to_string()),
}),
kv_records: vec![VaultKvRecordReference {
key_id: "object-key".to_string(),
kv_version: 4,
}],
}
}
fn transit_manifest_unsealed() -> BackupManifest {
BackupManifest {
backend: BackupBackendKind::VaultTransit,
at_rest_protection: AtRestProtection::ExternalNonExportable,
responsibility: BackupResponsibility::MetadataPlusExternalRoot,
artifacts: vec![artifact(ArtifactKind::KeyMetadata, "vault/records/object-key.json.enc")],
local_kdf: None,
external_references: Some(transit_references()),
..local_manifest_unsealed()
}
}
#[test]
fn vault_bundle_round_trips_with_external_references() {
let sealed = seal(transit_manifest_unsealed());
let bytes = sealed.encode().expect("encoding should succeed");
let decoded = BackupManifest::decode(&bytes).expect("decoding should succeed");
assert_eq!(decoded, sealed);
}
#[test]
fn vault_backend_requires_external_references() {
let manifest = BackupManifest {
external_references: None,
..transit_manifest_unsealed()
};
expect_corrupted(
BackupManifest::decode(&serde_json::to_vec(&seal(manifest)).expect("serialize")),
"must carry external Vault references",
);
}
#[test]
fn non_vault_backend_rejects_external_references() {
let manifest = BackupManifest {
external_references: Some(transit_references()),
..local_manifest_unsealed()
};
expect_corrupted(
BackupManifest::decode(&serde_json::to_vec(&seal(manifest)).expect("serialize")),
"not valid for backend",
);
}
#[test]
fn external_reference_contradictions_fail_closed() {
let cases: [(&str, Box<dyn Fn(&mut VaultExternalReferences)>); 7] = [
("cluster_id", Box::new(|refs| refs.cluster_id.clear())),
("kv_mount", Box::new(|refs| refs.kv_mount.clear())),
("namespace must not be empty", Box::new(|refs| refs.namespace = Some(String::new()))),
(
"must reference the Transit key",
Box::new(|refs: &mut VaultExternalReferences| refs.transit = None),
),
(
"at least key version 1",
Box::new(|refs: &mut VaultExternalReferences| {
if let Some(transit) = refs.transit.as_mut() {
transit.required_min_version = 0;
}
}),
),
(
"below the required minimum",
Box::new(|refs: &mut VaultExternalReferences| {
if let Some(transit) = refs.transit.as_mut() {
transit.current_version = 1;
}
}),
),
(
"appears more than once",
Box::new(|refs: &mut VaultExternalReferences| {
let first = refs.kv_records[0].clone();
refs.kv_records.push(first);
}),
),
];
for (needle, mutate) in cases {
let mut references = transit_references();
mutate(&mut references);
let manifest = BackupManifest {
external_references: Some(references),
..transit_manifest_unsealed()
};
expect_corrupted(BackupManifest::decode(&serde_json::to_vec(&seal(manifest)).expect("serialize")), needle);
}
}
/// The reference schema is the only place a Vault coordinate reaches a
/// bundle, so its field set is pinned: a credential-carrying field could
/// only appear by editing this assertion.
#[test]
fn external_reference_field_set_is_frozen() {
let value = serde_json::to_value(transit_references()).expect("serialize");
let mut top: Vec<&str> = value.as_object().expect("object").keys().map(String::as_str).collect();
top.sort_unstable();
assert_eq!(
top,
[
"cluster_id",
"kv_mount",
"kv_path_prefix",
"kv_records",
"namespace",
"transit"
]
);
let mut transit: Vec<&str> = value["transit"]
.as_object()
.expect("transit object")
.keys()
.map(String::as_str)
.collect();
transit.sort_unstable();
assert_eq!(
transit,
[
"current_version",
"key_name",
"mount_path",
"native_snapshot_reference",
"required_min_version"
]
);
let mut record: Vec<&str> = value["kv_records"][0]
.as_object()
.expect("record object")
.keys()
.map(String::as_str)
.collect();
record.sort_unstable();
assert_eq!(record, ["key_id", "kv_version"]);
}
}
+13 -5
View File
@@ -17,9 +17,11 @@
//! The contract side defines the versioned backup manifest, the per-backend
//! responsibility matrix, typed failure modes, and the restore dry-run
//! report. [`local_export`] implements the producer side and
//! [`local_restore`] the consumer side for the Local backend as
//! crate-internal APIs; the admin API builds on these pieces in follow-up
//! changes.
//! [`local_restore`] the consumer side for the Local backend;
//! [`vault_restore`] orchestrates the consumer side for the Vault backends,
//! whose cryptographic root is restored by Vault's own disaster-recovery
//! flow. All are crate-internal APIs; the admin API builds on these pieces in
//! follow-up changes.
//!
//! # Bundle model
//!
@@ -54,6 +56,7 @@ mod error;
pub mod local_export;
pub mod local_restore;
mod manifest;
pub mod vault_restore;
pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
pub use dry_run::{
@@ -62,7 +65,7 @@ pub use dry_run::{
pub use error::BackupError;
pub use local_export::{
BackupKek, LOCAL_BUNDLE_MANIFEST_FILE, LocalBackupExportRequest, decrypt_bundle_artifact, export_local_backup,
read_local_bundle_manifest,
read_bundle_manifest, read_local_bundle_manifest,
};
pub use local_restore::{
LocalRestoreReport, LocalRestoreRequest, RestoreConflictPolicy, abort_local_restore, dry_run_local_restore,
@@ -70,5 +73,10 @@ pub use local_restore::{
};
pub use manifest::{
AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupKekDescriptor, BackupManifest, CompletenessState, ContentDigest,
DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot,
DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot, VaultExternalReferences, VaultKvRecordReference,
VaultTransitReference,
};
pub use vault_restore::{
VaultRestoreClient, VaultRestoreMismatch, VaultRestoreReport, VaultRestoreRequest, VaultRestoreSequence, VaultRestoreStage,
VaultRestoreTarget, abort_vault_restore, dry_run_vault_restore, restore_vault_backup,
};
File diff suppressed because it is too large Load Diff
+38 -306
View File
@@ -16,84 +16,11 @@
use crate::types::KeyMetadata;
use moka::future::Cache;
use moka::notification::RemovalCause;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
/// Default lifetime of a cached key metadata entry.
const DEFAULT_METADATA_TTL: Duration = Duration::from_secs(300);
// ---------------------------------------------------------------------------
// Metrics
//
// Lookup outcomes and removals are counted here because moka exposes neither
// hit nor miss counts; the numbers below are the cache's own, not derived.
// Label values are exclusively static strings (lookup result, removal cause) —
// key identifiers and key metadata must never reach a metric label.
// ---------------------------------------------------------------------------
/// Counter: key metadata lookups, by `result` (`hit` or `miss`).
const METRIC_CACHE_LOOKUPS_TOTAL: &str = "rustfs_kms_metadata_cache_lookups_total";
/// Counter: entries dropped from the cache, by `cause` (`expired`, `size`,
/// `explicit`, `replaced`); only `expired` and `size` are true evictions, the
/// rest are invalidations driven by key lifecycle operations.
const METRIC_CACHE_EVICTIONS_TOTAL: &str = "rustfs_kms_metadata_cache_evictions_total";
/// Gauge: entries currently held by the cache. Republished whenever the entry
/// set can have changed — every write path, plus the lookups that miss, since
/// TTL expiry drops entries without any write taking place.
const METRIC_CACHE_ENTRIES: &str = "rustfs_kms_metadata_cache_entries";
/// Register metric descriptions once per process.
fn describe_metrics() {
static DESCRIBE: std::sync::Once = std::sync::Once::new();
DESCRIBE.call_once(|| {
metrics::describe_counter!(METRIC_CACHE_LOOKUPS_TOTAL, "Total KMS key metadata cache lookups, by hit or miss");
metrics::describe_counter!(
METRIC_CACHE_EVICTIONS_TOTAL,
"Total entries dropped from the KMS key metadata cache, by removal cause"
);
metrics::describe_gauge!(METRIC_CACHE_ENTRIES, "Entries currently held by the KMS key metadata cache");
});
}
fn removal_cause_label(cause: RemovalCause) -> &'static str {
match cause {
RemovalCause::Expired => "expired",
RemovalCause::Explicit => "explicit",
RemovalCause::Replaced => "replaced",
RemovalCause::Size => "size",
}
}
/// Cumulative cache counters. Shared with moka's eviction listener, which runs
/// outside any `&self` borrow, hence the `Arc` and the atomics.
#[derive(Debug, Default)]
struct CacheCounters {
hits: AtomicU64,
misses: AtomicU64,
evictions: AtomicU64,
}
/// Snapshot of the KMS key metadata cache counters.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct KmsCacheStats {
/// Entries currently held. Eventually consistent: moka applies pending
/// maintenance lazily.
pub entries: u64,
/// Lookups served from the cache since process start.
pub hits: u64,
/// Lookups that fell through to the backend since process start.
pub misses: u64,
/// Entries dropped since process start, whatever the cause (expiry,
/// capacity, invalidation, replacement).
pub evictions: u64,
}
/// KMS cache for storing frequently accessed keys and metadata
pub struct KmsCache {
key_metadata_cache: Cache<String, KeyMetadata>,
counters: Arc<CacheCounters>,
}
impl KmsCache {
@@ -106,26 +33,11 @@ impl KmsCache {
/// A new instance of `KmsCache`
///
pub fn new(capacity: u64) -> Self {
Self::with_ttl(capacity, DEFAULT_METADATA_TTL)
}
/// Create a new KMS cache with an explicit metadata time-to-live
fn with_ttl(capacity: u64, metadata_ttl: Duration) -> Self {
describe_metrics();
let counters = Arc::new(CacheCounters::default());
let eviction_counters = Arc::clone(&counters);
Self {
key_metadata_cache: Cache::builder()
.max_capacity(capacity)
.time_to_live(metadata_ttl)
.eviction_listener(move |_key: Arc<String>, _metadata: KeyMetadata, cause: RemovalCause| {
eviction_counters.evictions.fetch_add(1, Ordering::Relaxed);
metrics::counter!(METRIC_CACHE_EVICTIONS_TOTAL, "cause" => removal_cause_label(cause)).increment(1);
})
.time_to_live(Duration::from_secs(300)) // 5 minutes default TTL
.build(),
counters,
}
}
@@ -138,26 +50,7 @@ impl KmsCache {
/// An `Option` containing the `KeyMetadata` if found, or `None` if not found
///
pub async fn get_key_metadata(&self, key_id: &str) -> Option<KeyMetadata> {
let cached = self.key_metadata_cache.get(key_id).await;
self.record_lookup(cached.is_some());
if cached.is_none() {
// TTL expiry is the one way the entry set shrinks without a write,
// and a miss is where it surfaces. moka expires lazily: the miss is
// reported at once, but the removal that decrements `entry_count`
// and reaches the eviction listener lands in the maintenance pass
// moka runs opportunistically on reads, on its own interval. So
// this converges the gauge within a housekeeping interval rather
// than on the first miss — enough to stop a cache that only ever
// expires, with no put, remove or clear to follow, from reporting a
// population that is gone. Forcing `run_pending_tasks` would
// tighten that at the cost of taking moka's maintenance lock on
// every miss, for a gauge `entry_count` only approximates anyway.
// Hits, the hot path, are left alone.
self.record_entry_count();
}
cached
self.key_metadata_cache.get(key_id).await
}
/// Put key metadata into cache
@@ -169,7 +62,6 @@ impl KmsCache {
pub async fn put_key_metadata(&mut self, key_id: &str, metadata: &KeyMetadata) {
self.key_metadata_cache.insert(key_id.to_string(), metadata.clone()).await;
self.key_metadata_cache.run_pending_tasks().await;
self.record_entry_count();
}
/// Remove key metadata from cache
@@ -179,11 +71,6 @@ impl KmsCache {
///
pub async fn remove_key_metadata(&mut self, key_id: &str) {
self.key_metadata_cache.remove(key_id).await;
// Flush maintenance so the removal notification and the entry gauge
// describe the cache as it is once this call returns.
self.key_metadata_cache.run_pending_tasks().await;
self.record_entry_count();
}
/// Clear all cached entries
@@ -192,35 +79,18 @@ impl KmsCache {
// Wait for invalidation to complete
self.key_metadata_cache.run_pending_tasks().await;
self.record_entry_count();
}
/// Get cache statistics
/// Get cache statistics (hit count, miss count)
///
/// # Returns
/// A [`KmsCacheStats`] snapshot of entry count, hits, misses and evictions
/// A tuple containing total entries and total misses
///
pub fn stats(&self) -> KmsCacheStats {
KmsCacheStats {
entries: self.key_metadata_cache.entry_count(),
hits: self.counters.hits.load(Ordering::Relaxed),
misses: self.counters.misses.load(Ordering::Relaxed),
evictions: self.counters.evictions.load(Ordering::Relaxed),
}
}
fn record_lookup(&self, hit: bool) {
let (counter, result) = if hit {
(&self.counters.hits, "hit")
} else {
(&self.counters.misses, "miss")
};
counter.fetch_add(1, Ordering::Relaxed);
metrics::counter!(METRIC_CACHE_LOOKUPS_TOTAL, "result" => result).increment(1);
}
fn record_entry_count(&self) {
metrics::gauge!(METRIC_CACHE_ENTRIES).set(self.key_metadata_cache.entry_count() as f64);
pub fn stats(&self) -> (u64, u64) {
(
self.key_metadata_cache.entry_count(),
0u64, // moka doesn't provide miss count directly
)
}
}
@@ -229,90 +99,37 @@ mod tests {
use super::*;
use crate::types::{KeyState, KeyUsage};
use jiff::Zoned;
use metrics_util::MetricKind;
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
use std::time::Duration;
#[derive(Debug, Clone)]
struct CacheInfo {
key_metadata_count: u64,
}
impl CacheInfo {
fn total_entries(&self) -> u64 {
self.key_metadata_count
}
}
impl KmsCache {
fn with_ttl_for_tests(capacity: u64, metadata_ttl: Duration) -> Self {
Self {
key_metadata_cache: Cache::builder().max_capacity(capacity).time_to_live(metadata_ttl).build(),
}
}
fn info_for_tests(&self) -> CacheInfo {
CacheInfo {
key_metadata_count: self.key_metadata_cache.entry_count(),
}
}
fn contains_key_metadata_for_tests(&self, key_id: &str) -> bool {
self.key_metadata_cache.contains_key(key_id)
}
}
fn test_metadata(key_id: &str) -> KeyMetadata {
KeyMetadata {
key_id: key_id.to_string(),
key_state: KeyState::Enabled,
key_usage: KeyUsage::EncryptDecrypt,
description: None,
creation_date: Zoned::now(),
deletion_date: None,
origin: "KMS".to_string(),
key_manager: "CUSTOMER".to_string(),
tags: std::collections::HashMap::new(),
}
}
type MetricEntry = (
metrics_util::CompositeKey,
Option<metrics::Unit>,
Option<metrics::SharedString>,
DebugValue,
);
/// Drive `test` on a current-thread runtime under a thread-local debugging
/// recorder and return its output plus one snapshot of everything emitted.
///
/// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
/// the recorded state, so taking it per assertion would leave every
/// assertion after the first with nothing to look at.
fn record_metrics<F, Fut, T>(test: F) -> (T, Vec<MetricEntry>)
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = T>,
{
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let output = metrics::with_local_recorder(&recorder, || {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("current-thread runtime must build");
runtime.block_on(test())
});
(output, snapshotter.snapshot().into_vec())
}
/// Sum of counters with `name` whose labels include every `(key, value)` pair.
fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 {
snapshot
.iter()
.filter_map(|(composite, _unit, _description, value)| {
let key = composite.key();
let matches = composite.kind() == MetricKind::Counter
&& key.name() == name
&& labels
.iter()
.all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected));
match (matches, value) {
(true, DebugValue::Counter(count)) => Some(*count),
_ => None,
}
})
.sum()
}
/// Last value recorded for the unlabelled gauge `name`.
fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
snapshot.iter().find_map(|(composite, _unit, _description, value)| {
let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
match (matches, value) {
(true, DebugValue::Gauge(gauge)) => Some(**gauge),
_ => None,
}
})
}
#[tokio::test]
async fn test_cache_operations() {
let mut cache = KmsCache::new(100);
@@ -337,16 +154,19 @@ mod tests {
assert_eq!(retrieved.expect("metadata should be cached").key_id, "test-key-1");
// Test cache info
assert_eq!(cache.stats().entries, 1);
let info = cache.info_for_tests();
assert_eq!(info.key_metadata_count, 1);
assert_eq!(info.total_entries(), 1);
// Test cache clearing
cache.clear().await;
assert_eq!(cache.stats().entries, 0);
let info_after_clear = cache.info_for_tests();
assert_eq!(info_after_clear.total_entries(), 0);
}
#[tokio::test]
async fn test_cache_with_custom_ttl() {
let mut cache = KmsCache::with_ttl(
let mut cache = KmsCache::with_ttl_for_tests(
100,
Duration::from_millis(100), // Short TTL for testing
);
@@ -397,92 +217,4 @@ mod tests {
assert!(cache.contains_key_metadata_for_tests("contains-test"));
}
#[test]
fn lookups_report_real_hit_and_miss_counts() {
let (stats, snapshot) = record_metrics(|| async {
let mut cache = KmsCache::new(100);
assert!(cache.get_key_metadata("absent").await.is_none());
cache.put_key_metadata("present", &test_metadata("present")).await;
assert!(cache.get_key_metadata("present").await.is_some());
assert!(cache.get_key_metadata("absent").await.is_none());
cache.stats()
});
assert_eq!(stats.hits, 1);
assert_eq!(stats.misses, 2);
assert_eq!(counter_value(&snapshot, METRIC_CACHE_LOOKUPS_TOTAL, &[("result", "hit")]), 1);
assert_eq!(counter_value(&snapshot, METRIC_CACHE_LOOKUPS_TOTAL, &[("result", "miss")]), 2);
}
#[test]
fn removals_report_their_cause_and_the_resulting_entry_count() {
let (stats, snapshot) = record_metrics(|| async {
let mut cache = KmsCache::new(100);
cache.put_key_metadata("key", &test_metadata("key")).await;
cache.put_key_metadata("key", &test_metadata("key")).await;
cache.remove_key_metadata("key").await;
cache.stats()
});
assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "replaced")]), 1);
assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "explicit")]), 1);
assert_eq!(stats.evictions, 2);
assert_eq!(stats.entries, 0);
assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(0.0));
}
#[test]
fn capacity_pressure_reports_size_evictions() {
let (stats, snapshot) = record_metrics(|| async {
let mut cache = KmsCache::with_ttl(1, DEFAULT_METADATA_TTL);
cache.put_key_metadata("first", &test_metadata("first")).await;
cache.put_key_metadata("second", &test_metadata("second")).await;
cache.stats()
});
assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "size")]), 1);
assert_eq!(stats.evictions, 1);
assert_eq!(stats.entries, 1);
assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(1.0));
}
/// Entries that expire are dropped outside every write path, so the gauge
/// has to be republished from the lookup that observes the expiry — nothing
/// else runs afterwards to correct it.
///
/// moka's clock is internal and cannot be driven by tokio's paused time,
/// hence the one short real sleep. The explicit `run_pending_tasks` stands
/// in for the maintenance moka schedules by itself on reads, so the test
/// does not depend on moka's internal 300ms housekeeping interval.
#[test]
fn expiry_without_further_writes_converges_the_entry_gauge() {
let ttl = Duration::from_millis(100);
let (stats, snapshot) = record_metrics(move || async move {
let mut cache = KmsCache::with_ttl(100, ttl);
cache.put_key_metadata("expiring", &test_metadata("expiring")).await;
assert_eq!(cache.stats().entries, 1);
tokio::time::sleep(Duration::from_millis(150)).await;
cache.key_metadata_cache.run_pending_tasks().await;
// The lookup that observes the expiry is the last thing to touch
// the cache: no put, remove or clear follows it.
assert!(cache.get_key_metadata("expiring").await.is_none());
cache.stats()
});
assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "expired")]), 1);
assert_eq!(stats.entries, 0);
assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(0.0));
}
}
-133
View File
@@ -34,8 +34,6 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
@@ -130,26 +128,6 @@ pub enum KmsBackend {
/// Static single-key backend that derives DEKs from a pre-configured key
#[serde(rename = "Static")]
Static,
/// AWS KMS backend: AWS is the cryptographic source of truth and owns key
/// state, versioning, and the deletion window.
#[serde(rename = "AWS", alias = "AwsKms")]
Aws,
}
impl KmsBackend {
/// Stable identifier for logs, metrics, and audit records.
///
/// External consumers key off these values, so treat them as a wire
/// contract rather than a rendering detail.
pub fn as_str(&self) -> &'static str {
match self {
KmsBackend::VaultKv2 => "vault-kv2",
KmsBackend::VaultTransit => "vault-transit",
KmsBackend::Local => "local",
KmsBackend::Static => "static",
KmsBackend::Aws => "aws",
}
}
}
/// Main KMS configuration
@@ -207,9 +185,6 @@ pub enum BackendConfig {
VaultTransit(Box<VaultTransitConfig>),
/// Static single-key backend configuration
Static(StaticConfig),
/// AWS KMS backend configuration
#[serde(rename = "AWS", alias = "AwsKms")]
Aws(Box<AwsKmsConfig>),
}
impl Default for BackendConfig {
@@ -225,7 +200,6 @@ impl fmt::Debug for BackendConfig {
Self::VaultKv2(config) => f.debug_tuple("VaultKv2").field(config).finish(),
Self::VaultTransit(config) => f.debug_tuple("VaultTransit").field(config).finish(),
Self::Static(config) => f.debug_tuple("Static").field(config).finish(),
Self::Aws(config) => f.debug_tuple("Aws").field(config).finish(),
}
}
}
@@ -551,25 +525,6 @@ impl Default for CacheConfig {
}
}
/// AWS KMS backend configuration.
///
/// Deliberately holds no credential material: the backend resolves credentials
/// through the standard `aws-config` provider chain (environment, shared
/// profile, container/IMDS role), so RustFS never stores, persists, or redacts
/// AWS secrets of its own.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct AwsKmsConfig {
/// AWS region hosting the KMS keys. When unset, the region is resolved by
/// the standard chain (`AWS_REGION`, profile, IMDS).
#[serde(default)]
pub region: Option<String>,
/// Override for the KMS endpoint, for local emulators and private
/// endpoints. Unset in production, where the SDK derives the regional
/// endpoint.
#[serde(default)]
pub endpoint_url: Option<String>,
}
impl KmsConfig {
/// Create a new KMS configuration for local backend (for development and testing only)
pub fn local(key_dir: PathBuf) -> Self {
@@ -679,29 +634,6 @@ impl KmsConfig {
}
}
/// Create a new KMS configuration for the AWS KMS backend.
///
/// Credentials are resolved by the standard `aws-config` provider chain;
/// only the region is configured here.
pub fn aws(region: Option<String>) -> Self {
Self {
backend: KmsBackend::Aws,
backend_config: BackendConfig::Aws(Box::new(AwsKmsConfig {
region,
endpoint_url: None,
})),
..Default::default()
}
}
/// Get the AWS configuration if backend is AWS KMS
pub fn aws_kms_config(&self) -> Option<&AwsKmsConfig> {
match &self.backend_config {
BackendConfig::Aws(config) => Some(config),
_ => None,
}
}
/// Set default key ID
pub fn with_default_key(mut self, key_id: String) -> Self {
self.default_key_id = Some(key_id);
@@ -858,25 +790,6 @@ impl KmsConfig {
// Validate that the key can be decoded (right length, valid base64)
config.decode_key()?;
}
BackendConfig::Aws(config) => {
if let Some(region) = &config.region
&& region.is_empty()
{
return Err(KmsError::configuration_error("AWS KMS region cannot be empty when set"));
}
if let Some(endpoint) = &config.endpoint_url {
if !endpoint.starts_with("http://") && !endpoint.starts_with("https://") {
return Err(KmsError::configuration_error("AWS KMS endpoint URL must use http or https scheme"));
}
// A plaintext endpoint override exposes every KMS request,
// including plaintext data keys, so it stays gated on the
// explicit development opt-in.
if endpoint.starts_with("http://") && !self.allow_insecure_dev_defaults {
return Err(development_default_error("AWS KMS endpoint URL must use https"));
}
}
}
}
// Validate cache configuration
@@ -898,7 +811,6 @@ impl KmsConfig {
"vault" | "vault-kv2" | "vault_kv2" => KmsBackend::VaultKv2,
"vault-transit" | "vault_transit" => KmsBackend::VaultTransit,
"static" => KmsBackend::Static,
"aws" | "aws-kms" | "aws_kms" => KmsBackend::Aws,
_ => return Err(KmsError::configuration_error(format!("Unknown KMS backend: {backend_type}"))),
};
}
@@ -1027,14 +939,6 @@ impl KmsConfig {
});
config.default_key_id = Some(key_id);
}
KmsBackend::Aws => {
// Only non-credential settings are read here; access keys,
// profiles, and role assumption stay with the aws-config chain.
config.backend_config = BackendConfig::Aws(Box::new(AwsKmsConfig {
region: get_env_opt_str(ENV_KMS_AWS_REGION),
endpoint_url: get_env_opt_str(ENV_KMS_AWS_ENDPOINT_URL),
}));
}
}
config.validate()?;
@@ -1636,43 +1540,6 @@ mod tests {
);
}
/// The AWS backend reads only non-credential settings from the
/// environment; access keys, profiles, and role assumption stay with the
/// aws-config provider chain.
#[test]
fn test_from_env_selects_aws_backend_without_credentials() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("aws")),
(ENV_KMS_AWS_REGION, Some("eu-central-1")),
(ENV_KMS_AWS_ENDPOINT_URL, None::<&str>),
],
|| {
let config = KmsConfig::from_env().expect("kms config should load from env");
assert_eq!(config.backend, KmsBackend::Aws);
let aws = config.aws_kms_config().expect("aws backend config");
assert_eq!(aws.region.as_deref(), Some("eu-central-1"));
assert_eq!(aws.endpoint_url, None);
},
);
}
/// A plaintext endpoint override exposes every KMS request, including the
/// plaintext data keys, so it stays behind the explicit development opt-in.
#[test]
fn test_from_env_rejects_plaintext_aws_endpoint() {
with_vars(
vec![
("RUSTFS_KMS_BACKEND", Some("aws")),
(ENV_KMS_AWS_REGION, Some("us-east-1")),
(ENV_KMS_AWS_ENDPOINT_URL, Some("http://localhost:4566")),
],
|| {
KmsConfig::from_env().expect_err("a plaintext AWS endpoint must be rejected by default");
},
);
}
#[test]
fn test_from_env_approle_requires_secret_id_or_file() {
with_vars(
+18 -379
View File
@@ -22,123 +22,18 @@
//! every node of a deployment concurrently — a key is only ever removed while
//! its (re-read) record is an expired pending deletion or a tombstone.
use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
use crate::backends::{ExpiredKeyRemoval, KmsBackend};
use crate::error::Result;
use crate::types::{KeyInfo, KeyStatus, ListKeysRequest, OperationContext};
use crate::types::{KeyStatus, ListKeysRequest};
use async_trait::async_trait;
use jiff::Zoned;
use std::sync::Arc;
use std::time::{Duration, Instant};
use std::time::Duration;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, warn};
/// How often the worker looks for expired pending deletions.
pub const DEFAULT_SWEEP_INTERVAL: Duration = Duration::from_secs(60);
// ---------------------------------------------------------------------------
// Metrics
//
// The sweep already pages through the whole key set, so everything below is
// derived from the pages it has in hand: observing the key lifecycle costs no
// extra backend call. Each metric is an aggregate — the gauges carry no labels
// at all and the counter only a static outcome — because a per-key label would
// carry key identifiers into the metric stream and grow the series count with
// the key set. "This key is overdue for rotation" is a threshold on the
// aggregate age and belongs in an alerting rule, not in a label.
// ---------------------------------------------------------------------------
/// Gauge: keys scheduled for deletion whose deadline has not passed, as of the
/// end of the last sweep that saw the whole key set.
const METRIC_PENDING_DELETION_KEYS: &str = "rustfs_kms_pending_deletion_keys";
/// Gauge: keys left tombstoned by an interrupted removal and still awaiting
/// the sweep, as of the end of the last sweep that saw the whole key set.
const METRIC_TOMBSTONE_KEYS: &str = "rustfs_kms_deletion_tombstone_keys";
/// Gauge: seconds since the least recently rotated usable key was rotated
/// (its creation time when it was never rotated); `0` when there are none.
const METRIC_OLDEST_ROTATION_AGE_SECONDS: &str = "rustfs_kms_oldest_key_rotation_age_seconds";
/// Counter: keys the sweep acted on, by `outcome` (`removed`, `blocked`,
/// `skipped`, `failed`).
const METRIC_SWEEP_KEYS_TOTAL: &str = "rustfs_kms_deletion_sweep_keys_total";
/// Register metric descriptions once per process.
fn describe_metrics() {
static DESCRIBE: std::sync::Once = std::sync::Once::new();
DESCRIBE.call_once(|| {
metrics::describe_gauge!(
METRIC_PENDING_DELETION_KEYS,
"KMS keys scheduled for deletion whose deadline has not passed yet"
);
metrics::describe_gauge!(
METRIC_TOMBSTONE_KEYS,
"KMS keys left tombstoned by an interrupted removal, still awaiting the deletion sweep"
);
metrics::describe_gauge!(
METRIC_OLDEST_ROTATION_AGE_SECONDS,
"Seconds since the least recently rotated usable KMS key was last rotated, counting from creation for keys that were never rotated"
);
metrics::describe_counter!(METRIC_SWEEP_KEYS_TOTAL, "Total keys acted on by the KMS deletion sweep, by outcome");
});
}
/// Aggregate view of the key set, accumulated while the sweep pages through it.
///
/// Counts and one maximum age only: these feed label-less gauges, so no key
/// identifier can reach the metric stream through them.
#[derive(Debug, Default, Clone, Copy, PartialEq)]
struct KeyCensus {
pending_deletion: usize,
tombstones: usize,
/// Longest time since last rotation across keys that are still usable,
/// counting from creation for keys that were never rotated. Keys on their
/// way out are excluded: they will never be rotated again, and would
/// otherwise pin the gauge high until the sweep finishes removing them.
oldest_rotation_age_seconds: f64,
}
impl KeyCensus {
fn observe(&mut self, key: &KeyInfo, now: &Zoned) {
match key.status {
KeyStatus::PendingDeletion => self.pending_deletion += 1,
KeyStatus::Deleted => self.tombstones += 1,
KeyStatus::Active | KeyStatus::Disabled => {
let rotated_at = key.rotated_at.as_ref().unwrap_or(&key.created_at);
self.oldest_rotation_age_seconds = self.oldest_rotation_age_seconds.max(seconds_between(rotated_at, now));
}
}
}
}
/// Seconds from `earlier` to `now`, clamped at zero so clock skew (or a
/// timestamp persisted by a node running ahead) cannot produce a negative age.
fn seconds_between(earlier: &Zoned, now: &Zoned) -> f64 {
(now.timestamp().as_second() - earlier.timestamp().as_second()).max(0) as f64
}
/// Publish one sweep's counters, plus the lifecycle gauges when `census` covers
/// the whole key set.
fn record_sweep(report: &SweepReport, census: Option<KeyCensus>) {
describe_metrics();
for (outcome, count) in [
("removed", report.removed.len()),
("blocked", report.blocked.len()),
("skipped", report.skipped),
("failed", report.failed),
] {
// Emitted even at zero so every outcome series exists from the first
// sweep on and a rate over it is defined.
metrics::counter!(METRIC_SWEEP_KEYS_TOTAL, "outcome" => outcome).increment(count as u64);
}
// A sweep that could not finish listing saw only part of the key set;
// publishing its counts would understate every gauge, so the previous
// (complete) values are left standing instead.
let Some(census) = census else { return };
metrics::gauge!(METRIC_PENDING_DELETION_KEYS).set(census.pending_deletion as f64);
metrics::gauge!(METRIC_TOMBSTONE_KEYS).set(census.tombstones as f64);
metrics::gauge!(METRIC_OLDEST_ROTATION_AGE_SECONDS).set(census.oldest_rotation_age_seconds);
}
/// Reports configuration that still references a KMS key.
///
/// Consulted before any material is destroyed; a non-empty result blocks the
@@ -173,8 +68,6 @@ pub(crate) struct DeletionWorker {
default_key_id: Option<String>,
reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
interval: Duration,
backend_kind: &'static str,
audit_sink: Option<Arc<dyn KmsAuditSink>>,
}
impl DeletionWorker {
@@ -182,24 +75,15 @@ impl DeletionWorker {
backend: Arc<dyn KmsBackend>,
default_key_id: Option<String>,
reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
backend_kind: &'static str,
) -> Self {
Self {
backend,
default_key_id,
reference_checker,
backend_kind,
audit_sink: None,
interval: DEFAULT_SWEEP_INTERVAL,
}
}
/// Audit each irreversible key removal to `sink`.
pub(crate) fn with_audit_sink(mut self, sink: Option<Arc<dyn KmsAuditSink>>) -> Self {
self.audit_sink = sink;
self
}
pub(crate) fn spawn(self, cancel: CancellationToken) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move { self.run(cancel).await })
}
@@ -230,15 +114,10 @@ impl DeletionWorker {
/// Run one sweep at the given time. Exposed separately so tests can drive
/// the expiry logic deterministically.
///
/// Also feeds the lifecycle gauges: every key it lists is counted into a
/// [`KeyCensus`] on the way past, so the observability comes out of the
/// pages the sweep already had to fetch.
pub(crate) async fn sweep(&self, now: &Zoned) -> SweepReport {
let mut report = SweepReport::default();
let mut census = KeyCensus::default();
let mut marker: Option<String> = None;
let listed_everything = loop {
loop {
let request = ListKeysRequest {
limit: Some(100),
marker: marker.clone(),
@@ -250,99 +129,54 @@ impl DeletionWorker {
Err(error) => {
warn!(%error, "KMS deletion sweep could not list keys");
report.failed += 1;
break false;
return report;
}
};
for key in &response.keys {
// Keys this sweep destroys are left out of the census: the
// gauges describe the key set as it stands once the sweep is
// done, not as it was when the page was listed.
let removed = if matches!(key.status, KeyStatus::PendingDeletion | KeyStatus::Deleted) {
self.process_key(&key.key_id, now, &mut report).await
} else {
false
};
if !removed {
census.observe(key, now);
if matches!(key.status, KeyStatus::PendingDeletion | KeyStatus::Deleted) {
self.process_key(&key.key_id, now, &mut report).await;
}
}
if !response.truncated {
break true;
break;
}
match response.next_marker {
Some(next_marker) => marker = Some(next_marker),
// Truncated without a marker: the rest of the key set is out
// of reach, so the census is incomplete.
None => break false,
None => break,
}
};
record_sweep(&report, listed_everything.then_some(census));
}
report
}
/// Handle one key that is on its way out; reports whether it was removed.
async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) -> bool {
async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) {
// Never remove a key that live configuration still points at. The
// default key check is built in; broader references (bucket
// encryption settings, ...) come from the injected checker.
if self.default_key_id.as_deref() == Some(key_id) {
warn!(key_id, "expired KMS key is still the default key; refusing removal");
report.blocked.push(key_id.to_string());
return false;
return;
}
if let Some(checker) = &self.reference_checker {
let references = checker.references(key_id).await;
if !references.is_empty() {
warn!(key_id, ?references, "expired KMS key is still referenced; refusing removal");
report.blocked.push(key_id.to_string());
return false;
return;
}
}
// The backend re-checks state and deadline under its own write
// synchronization, so a cancellation racing this sweep wins there.
let started = Instant::now();
let outcome = self.backend.remove_expired_key(key_id, now).await;
match &outcome {
Ok(ExpiredKeyRemoval::Removed) => {
report.removed.push(key_id.to_string());
self.audit_removal(key_id, started, &outcome);
true
}
Ok(ExpiredKeyRemoval::StateChanged | ExpiredKeyRemoval::NotExpired) => {
report.skipped += 1;
false
}
match self.backend.remove_expired_key(key_id, now).await {
Ok(ExpiredKeyRemoval::Removed) => report.removed.push(key_id.to_string()),
Ok(ExpiredKeyRemoval::StateChanged | ExpiredKeyRemoval::NotExpired) => report.skipped += 1,
Err(error) => {
warn!(key_id, %error, "failed to remove expired KMS key; will retry next sweep");
report.failed += 1;
self.audit_removal(key_id, started, &outcome);
false
}
}
}
/// Record an attempted destruction of key material.
///
/// Only attempts that reached the backend are audited: a key the sweep
/// declined to touch (not yet due, still referenced, state changed under
/// us) was never at risk, and recording it as a deletion event would
/// drown the real ones.
fn audit_removal(&self, key_id: &str, started: Instant, outcome: &Result<ExpiredKeyRemoval>) {
let Some(sink) = self.audit_sink.as_ref() else {
return;
};
// Removal runs on a background sweep, so there is no request
// principal to attribute it to.
let context = OperationContext::internal();
sink.emit(
KmsAuditRecord::new(KmsAuditOperation::DeleteKey, &context, self.backend_kind)
.with_key_id(Some(key_id))
.with_latency(started.elapsed())
.with_result(outcome),
);
}
}
#[cfg(test)]
@@ -382,7 +216,7 @@ mod tests {
}
fn worker(backend: Arc<LocalKmsBackend>) -> DeletionWorker {
DeletionWorker::new(backend, None, None, "local")
DeletionWorker::new(backend, None, None)
}
fn after_window() -> Zoned {
@@ -473,7 +307,7 @@ mod tests {
schedule(&backend, &key_id).await;
// Blocked while it is the configured default key.
let as_default = DeletionWorker::new(backend.clone(), Some(key_id.clone()), None, "local");
let as_default = DeletionWorker::new(backend.clone(), Some(key_id.clone()), None);
let report = as_default.sweep(&after_window()).await;
assert_eq!(report.blocked, vec![key_id.clone()]);
assert!(report.removed.is_empty());
@@ -483,7 +317,6 @@ mod tests {
backend.clone(),
None,
Some(Arc::new(StaticReferences(vec!["bucket:sse-bucket".to_string()]))),
"local",
);
let report = with_references.sweep(&after_window()).await;
assert_eq!(report.blocked, vec![key_id.clone()]);
@@ -494,53 +327,11 @@ mod tests {
.expect("blocked key must still exist");
// Removed once nothing references it anymore.
let unreferenced = DeletionWorker::new(backend.clone(), None, Some(Arc::new(StaticReferences(Vec::new()))), "local");
let unreferenced = DeletionWorker::new(backend.clone(), None, Some(Arc::new(StaticReferences(Vec::new()))));
let report = unreferenced.sweep(&after_window()).await;
assert_eq!(report.removed, vec![key_id.clone()]);
}
#[tokio::test]
async fn only_attempted_removals_are_audited() {
#[derive(Default)]
struct CapturingSink {
records: std::sync::Mutex<Vec<KmsAuditRecord>>,
}
impl KmsAuditSink for CapturingSink {
fn emit(&self, record: KmsAuditRecord) {
self.records.lock().expect("sink lock should not be poisoned").push(record);
}
}
let temp_dir = tempfile::tempdir().expect("temp dir");
let backend = local_backend(&temp_dir).await;
let key_id = create_key(&backend, "audited-removal").await;
schedule(&backend, &key_id).await;
let sink = Arc::new(CapturingSink::default());
let worker = DeletionWorker::new(backend.clone(), None, None, "local").with_audit_sink(Some(sink.clone()));
// A key that is not yet due was never at risk, so it produces no record.
worker.sweep(&Zoned::now()).await;
assert!(
sink.records.lock().expect("sink lock").is_empty(),
"a key the sweep declined to touch must not be audited as a deletion"
);
worker.sweep(&after_window()).await;
let records = sink.records.lock().expect("sink lock");
assert_eq!(records.len(), 1, "the removal should be audited exactly once");
let record = &records[0];
assert_eq!(record.operation, crate::audit::KmsAuditOperation::DeleteKey);
assert_eq!(record.event, rustfs_s3_types::EventName::KmsKeyDeleted);
assert_eq!(record.outcome, crate::audit::KmsAuditOutcome::Success);
assert_eq!(record.key_id.as_deref(), Some(key_id.as_str()));
assert_eq!(record.backend, "local");
// Background work has no request principal to attribute.
assert_eq!(record.principal, OperationContext::INTERNAL_PRINCIPAL);
}
#[tokio::test]
async fn deadline_survives_backend_restart_and_sweep_completes_it() {
let temp_dir = tempfile::tempdir().expect("temp dir");
@@ -604,156 +395,4 @@ mod tests {
cancel.cancel();
task.await.expect("worker task must stop after cancellation");
}
// -- Metric emission ----------------------------------------------------
use metrics_util::MetricKind;
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
type MetricEntry = (
metrics_util::CompositeKey,
Option<metrics::Unit>,
Option<metrics::SharedString>,
DebugValue,
);
/// Run `test` on a current-thread runtime under a debugging recorder and
/// return one snapshot of everything it emitted.
///
/// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
/// the recorded state, so taking it per assertion would only show the
/// first assertion any data.
fn record_metrics<Out>(
test: impl FnOnce() -> std::pin::Pin<Box<dyn std::future::Future<Output = Out>>>,
) -> (Vec<MetricEntry>, Out) {
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let out = metrics::with_local_recorder(&recorder, || {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("current-thread runtime must build");
runtime.block_on(test())
});
(snapshotter.snapshot().into_vec(), out)
}
fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
snapshot.iter().find_map(|(composite, _unit, _description, value)| {
let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
match (matches, value) {
(true, DebugValue::Gauge(value)) => Some(value.into_inner()),
_ => None,
}
})
}
fn counter_value(snapshot: &[MetricEntry], name: &str, outcome: &str) -> u64 {
snapshot
.iter()
.filter_map(|(composite, _unit, _description, value)| {
let matches = composite.kind() == MetricKind::Counter
&& composite.key().name() == name
&& composite
.key()
.labels()
.any(|label| label.key() == "outcome" && label.value() == outcome);
match (matches, value) {
(true, DebugValue::Counter(count)) => Some(*count),
_ => None,
}
})
.sum()
}
fn key_info(key_id: &str, status: KeyStatus, created_at: Zoned, rotated_at: Option<Zoned>) -> KeyInfo {
KeyInfo {
key_id: key_id.to_string(),
description: None,
algorithm: "AES-256".to_string(),
usage: KeyUsage::EncryptDecrypt,
status,
version: 1,
metadata: std::collections::HashMap::new(),
tags: std::collections::HashMap::new(),
created_at,
rotated_at,
created_by: None,
}
}
#[test]
fn census_ages_from_rotation_and_ignores_departing_keys() {
let now = Zoned::now();
let day = Duration::from_secs(86400);
let mut census = KeyCensus::default();
// Rotation beats creation as the age baseline...
census.observe(
&key_info("rotated", KeyStatus::Active, now.clone() - 30 * day, Some(now.clone() - 2 * day)),
&now,
);
// ...and a key that was never rotated ages from its creation.
census.observe(&key_info("never-rotated", KeyStatus::Disabled, now.clone() - 5 * day, None), &now);
// Keys on their way out only ever move the counts: they will not be
// rotated again, so their age must not drive the rotation gauge.
census.observe(&key_info("pending", KeyStatus::PendingDeletion, now.clone() - 400 * day, None), &now);
census.observe(&key_info("tombstone", KeyStatus::Deleted, now.clone() - 400 * day, None), &now);
assert_eq!(census.pending_deletion, 1);
assert_eq!(census.tombstones, 1);
let expected = 5.0 * 86400.0;
assert!(
(census.oldest_rotation_age_seconds - expected).abs() < 1.0,
"expected the never-rotated key to set the age, got {}",
census.oldest_rotation_age_seconds
);
}
#[test]
fn sweep_publishes_lifecycle_gauges_without_key_labels() {
let (snapshot, key_ids) = record_metrics(|| {
Box::pin(async {
let temp_dir = tempfile::tempdir().expect("temp dir");
let backend = local_backend(&temp_dir).await;
let live = create_key(&backend, "live-key").await;
let doomed = create_key(&backend, "doomed-key").await;
schedule(&backend, &doomed).await;
// Three days in, still inside the seven-day window: the sweep
// observes the scheduled key instead of removing it.
let report = worker(backend.clone())
.sweep(&(Zoned::now() + Duration::from_secs(3 * 86400)))
.await;
assert_eq!(report.skipped, 1);
assert!(report.removed.is_empty());
vec![live, doomed]
})
});
assert_eq!(gauge_value(&snapshot, METRIC_PENDING_DELETION_KEYS), Some(1.0));
assert_eq!(gauge_value(&snapshot, METRIC_TOMBSTONE_KEYS), Some(0.0));
let age = gauge_value(&snapshot, METRIC_OLDEST_ROTATION_AGE_SECONDS).expect("rotation age gauge must be published");
// Both keys were just created, and only the usable one counts, so the
// reported age is the sweep's own offset into the future.
assert!(
(age - 3.0 * 86400.0).abs() < 60.0,
"expected the sweep offset as the rotation age, got {age}"
);
assert_eq!(counter_value(&snapshot, METRIC_SWEEP_KEYS_TOTAL, "skipped"), 1);
assert_eq!(counter_value(&snapshot, METRIC_SWEEP_KEYS_TOTAL, "removed"), 0);
for (composite, ..) in &snapshot {
for label in composite.key().labels() {
for key_id in &key_ids {
assert!(
!label.value().contains(key_id.as_str()),
"metric {} leaked a key identifier through label {}",
composite.key().name(),
label.key()
);
}
}
}
}
}
-5
View File
@@ -65,7 +65,6 @@
// Core modules
pub mod api_types;
pub mod audit;
pub mod backends;
pub mod backup;
mod cache;
@@ -75,7 +74,6 @@ mod encryption;
mod error;
pub mod manager;
mod policy;
pub mod probe;
pub mod service;
pub mod service_manager;
mod time_serde;
@@ -88,14 +86,11 @@ pub use api_types::{
StartKmsResponse, StopKmsResponse, TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse,
UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
};
pub use audit::{KmsAuditOperation, KmsAuditOutcome, KmsAuditRecord, KmsAuditSink, redact_encryption_context};
pub use cache::KmsCacheStats;
pub use config::*;
pub use deletion_worker::DeletionReferenceChecker;
pub use encryption::is_data_key_envelope;
pub use error::{KmsError, KmsUnavailableError, Result};
pub use manager::KmsManager;
pub use probe::{ProbeFailureKind, ProbeResult, ProbeStatus};
pub use service::{DataKey, ObjectEncryptionService};
pub use service_manager::{
KmsServiceManager, KmsServiceStatus, KmsStartOutcome, get_global_encryption_service, get_global_kms_service_manager,
+16 -669
View File
@@ -14,19 +14,16 @@
//! KMS manager for handling key operations and backend coordination
use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
use crate::backends::KmsBackend;
use crate::cache::{KmsCache, KmsCacheStats};
use crate::cache::KmsCache;
use crate::config::KmsConfig;
use crate::error::Result;
use crate::types::{
CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse, DecryptRequest, DecryptResponse,
DeleteKeyRequest, DeleteKeyResponse, DescribeKeyRequest, DescribeKeyResponse, EncryptRequest, EncryptResponse,
GenerateDataKeyRequest, GenerateDataKeyResponse, ListKeysRequest, ListKeysResponse, OperationContext,
GenerateDataKeyRequest, GenerateDataKeyResponse, ListKeysRequest, ListKeysResponse,
};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Instant;
use tokio::sync::RwLock;
/// KMS Manager coordinates operations between backends and caching
@@ -36,8 +33,6 @@ pub struct KmsManager {
cache: Arc<RwLock<KmsCache>>,
default_key_id: Option<String>,
enable_cache: bool,
backend_kind: &'static str,
audit_sink: Option<Arc<dyn KmsAuditSink>>,
}
impl KmsManager {
@@ -49,72 +44,16 @@ impl KmsManager {
cache,
default_key_id: config.default_key_id,
enable_cache: config.enable_cache,
backend_kind: config.backend.as_str(),
audit_sink: None,
}
}
/// Send an audit record for every management operation to `sink`.
///
/// Without a sink the manager builds no records at all, so a deployment
/// that does not consume KMS audit records is unaffected.
pub fn with_audit_sink(mut self, sink: Arc<dyn KmsAuditSink>) -> Self {
self.audit_sink = Some(sink);
self
}
/// Get the default key ID if configured
pub fn get_default_key_id(&self) -> Option<&String> {
self.default_key_id.as_ref()
}
/// Emit an audit record for a completed management operation.
///
/// Called after the operation resolved, so nothing here can change its
/// result; the record is built only when a sink is installed.
fn audit<T>(
&self,
operation: KmsAuditOperation,
context: &OperationContext,
key_id: Option<&str>,
started: Instant,
result: &Result<T>,
) {
let Some(sink) = self.audit_sink.as_ref() else {
return;
};
sink.emit(
KmsAuditRecord::new(operation, context, self.backend_kind)
.with_key_id(key_id)
.with_latency(started.elapsed())
.with_result(result),
);
}
/// Create a new master key
///
/// Audited as an internal operation; callers serving an authenticated
/// request should use [`Self::create_key_with_context`].
pub async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
self.create_key_with_context(request, &OperationContext::internal()).await
}
/// Create a new master key on behalf of `context`'s principal
pub async fn create_key_with_context(
&self,
request: CreateKeyRequest,
context: &OperationContext,
) -> Result<CreateKeyResponse> {
let started = Instant::now();
let key_name = request.key_name.clone();
let result = self.create_key_inner(request).await;
let key_id = result.as_ref().ok().map(|r| r.key_id.as_str()).or(key_name.as_deref());
self.audit(KmsAuditOperation::CreateKey, context, key_id, started, &result);
result
}
async fn create_key_inner(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
let response = self.backend.create_key(request).await?;
// Cache the key metadata if enabled
@@ -145,27 +84,7 @@ impl KmsManager {
}
/// Describe a key
///
/// Audited as an internal operation; callers serving an authenticated
/// request should use [`Self::describe_key_with_context`].
pub async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
self.describe_key_with_context(request, &OperationContext::internal()).await
}
/// Describe a key on behalf of `context`'s principal
pub async fn describe_key_with_context(
&self,
request: DescribeKeyRequest,
context: &OperationContext,
) -> Result<DescribeKeyResponse> {
let started = Instant::now();
let key_id = request.key_id.clone();
let result = self.describe_key_inner(request).await;
self.audit(KmsAuditOperation::DescribeKey, context, Some(&key_id), started, &result);
result
}
async fn describe_key_inner(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
// Check cache first if enabled
if self.enable_cache {
let cache = self.cache.read().await;
@@ -190,24 +109,12 @@ impl KmsManager {
}
/// List keys
///
/// Audited as an internal operation; callers serving an authenticated
/// request should use [`Self::list_keys_with_context`].
pub async fn list_keys(&self, request: ListKeysRequest) -> Result<ListKeysResponse> {
self.list_keys_with_context(request, &OperationContext::internal()).await
self.backend.list_keys(request).await
}
/// List keys on behalf of `context`'s principal
pub async fn list_keys_with_context(&self, request: ListKeysRequest, context: &OperationContext) -> Result<ListKeysResponse> {
let started = Instant::now();
let result = self.backend.list_keys(request).await;
// Listing spans keys, so the record carries no key id.
self.audit(KmsAuditOperation::ListKeys, context, None, started, &result);
result
}
/// Get cache statistics, or `None` when caching is disabled
pub async fn cache_stats(&self) -> Option<KmsCacheStats> {
/// Get cache statistics
pub async fn cache_stats(&self) -> Option<(u64, u64)> {
if self.enable_cache {
let cache = self.cache.read().await;
Some(cache.stats())
@@ -226,27 +133,7 @@ impl KmsManager {
}
/// Delete a key
///
/// Audited as an internal operation; callers serving an authenticated
/// request should use [`Self::delete_key_with_context`].
pub async fn delete_key(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
self.delete_key_with_context(request, &OperationContext::internal()).await
}
/// Delete a key on behalf of `context`'s principal
pub async fn delete_key_with_context(
&self,
request: DeleteKeyRequest,
context: &OperationContext,
) -> Result<DeleteKeyResponse> {
let started = Instant::now();
let key_id = request.key_id.clone();
let result = self.delete_key_inner(request).await;
self.audit(KmsAuditOperation::ScheduleKeyDeletion, context, Some(&key_id), started, &result);
result
}
async fn delete_key_inner(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
let response = self.backend.delete_key(request).await?;
// Remove from cache if enabled and key is being deleted
@@ -259,28 +146,7 @@ impl KmsManager {
}
/// Cancel key deletion
///
/// Audited as an internal operation; callers serving an authenticated
/// request should use [`Self::cancel_key_deletion_with_context`].
pub async fn cancel_key_deletion(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> {
self.cancel_key_deletion_with_context(request, &OperationContext::internal())
.await
}
/// Cancel key deletion on behalf of `context`'s principal
pub async fn cancel_key_deletion_with_context(
&self,
request: CancelKeyDeletionRequest,
context: &OperationContext,
) -> Result<CancelKeyDeletionResponse> {
let started = Instant::now();
let key_id = request.key_id.clone();
let result = self.cancel_key_deletion_inner(request).await;
self.audit(KmsAuditOperation::CancelKeyDeletion, context, Some(&key_id), started, &result);
result
}
async fn cancel_key_deletion_inner(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> {
let response = self.backend.cancel_key_deletion(request).await?;
// Update cache if enabled
@@ -293,79 +159,22 @@ impl KmsManager {
}
/// Enable a disabled key
///
/// Audited as an internal operation; callers serving an authenticated
/// request should use [`Self::enable_key_with_context`].
pub async fn enable_key(&self, key_id: &str) -> Result<()> {
self.enable_key_with_context(key_id, &OperationContext::internal()).await
}
/// Enable a disabled key on behalf of `context`'s principal
pub async fn enable_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
let started = Instant::now();
let result = self.backend.enable_key(key_id).await;
if result.is_ok() {
self.invalidate_cached_metadata(key_id).await;
}
self.audit(KmsAuditOperation::EnableKey, context, Some(key_id), started, &result);
result
self.backend.enable_key(key_id).await?;
self.invalidate_cached_metadata(key_id).await;
Ok(())
}
/// Disable a key; existing data remains decryptable
///
/// Audited as an internal operation; callers serving an authenticated
/// request should use [`Self::disable_key_with_context`].
pub async fn disable_key(&self, key_id: &str) -> Result<()> {
self.disable_key_with_context(key_id, &OperationContext::internal()).await
}
/// Disable a key on behalf of `context`'s principal
pub async fn disable_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
let started = Instant::now();
let result = self.backend.disable_key(key_id).await;
if result.is_ok() {
self.invalidate_cached_metadata(key_id).await;
}
self.audit(KmsAuditOperation::DisableKey, context, Some(key_id), started, &result);
result
self.backend.disable_key(key_id).await?;
self.invalidate_cached_metadata(key_id).await;
Ok(())
}
/// Rotate a key to a new version
///
/// Audited as an internal operation; callers serving an authenticated
/// request should use [`Self::rotate_key_with_context`].
pub async fn rotate_key(&self, key_id: &str) -> Result<()> {
self.rotate_key_with_context(key_id, &OperationContext::internal()).await
}
/// Rotate a key on behalf of `context`'s principal
pub async fn rotate_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
let started = Instant::now();
let result = self.backend.rotate_key(key_id).await;
if result.is_ok() {
self.invalidate_cached_metadata(key_id).await;
}
self.audit(KmsAuditOperation::RotateKey, context, Some(key_id), started, &result);
result
}
/// Replace a key's description; `None` clears it
pub async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
self.backend.update_key_description(key_id, description).await?;
self.invalidate_cached_metadata(key_id).await;
Ok(())
}
/// Add or overwrite key tags, leaving every other tag untouched
pub async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
self.backend.tag_key(key_id, tags).await?;
self.invalidate_cached_metadata(key_id).await;
Ok(())
}
/// Remove key tags; tags that are not set are ignored
pub async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
self.backend.untag_key(key_id, tag_keys).await?;
self.backend.rotate_key(key_id).await?;
self.invalidate_cached_metadata(key_id).await;
Ok(())
}
@@ -399,470 +208,11 @@ impl KmsManager {
#[cfg(test)]
mod tests {
use super::*;
use crate::audit::KmsAuditOutcome;
use crate::backends::local::LocalKmsBackend;
use crate::error::KmsError;
use crate::types::{KeyMetadata, KeySpec, KeyState, KeyUsage};
use async_trait::async_trait;
use base64::Engine as _;
use jiff::Zoned;
use crate::types::{KeySpec, KeyState, KeyUsage};
use std::collections::HashMap;
use std::sync::Mutex;
use tempfile::tempdir;
/// Sink that keeps every record so tests can assert on the audit trail.
#[derive(Default)]
struct CapturingSink {
records: Mutex<Vec<KmsAuditRecord>>,
}
impl KmsAuditSink for CapturingSink {
fn emit(&self, record: KmsAuditRecord) {
self.records
.lock()
.expect("audit records lock should not be poisoned")
.push(record);
}
}
impl CapturingSink {
fn records(&self) -> Vec<KmsAuditRecord> {
self.records
.lock()
.expect("audit records lock should not be poisoned")
.clone()
}
fn take_one(&self) -> KmsAuditRecord {
let mut records = self.records.lock().expect("audit records lock should not be poisoned");
assert_eq!(records.len(), 1, "expected exactly one audit record, got {records:?}");
records.remove(0)
}
}
/// Backend whose management operations all succeed or all fail, so a
/// single test can drive every audited operation down both paths —
/// including operations no real backend supports on both.
struct ScriptedBackend {
failure: Option<KmsError>,
}
impl ScriptedBackend {
fn succeeding() -> Self {
Self { failure: None }
}
fn failing(failure: KmsError) -> Self {
Self { failure: Some(failure) }
}
fn check(&self) -> Result<()> {
match &self.failure {
Some(failure) => Err(failure.clone()),
None => Ok(()),
}
}
fn metadata(key_id: &str) -> KeyMetadata {
KeyMetadata {
key_id: key_id.to_string(),
key_state: KeyState::Enabled,
key_usage: KeyUsage::EncryptDecrypt,
description: None,
creation_date: Zoned::now(),
deletion_date: None,
origin: "RUSTFS_KMS".to_string(),
key_manager: "RUSTFS".to_string(),
tags: HashMap::new(),
}
}
}
#[async_trait]
impl KmsBackend for ScriptedBackend {
async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
self.check()?;
let key_id = request.key_name.unwrap_or_else(|| "scripted-key".to_string());
Ok(CreateKeyResponse {
key_metadata: Self::metadata(&key_id),
key_id,
})
}
async fn encrypt(&self, _request: EncryptRequest) -> Result<EncryptResponse> {
unimplemented!("data plane is not audited by the manager")
}
async fn decrypt(&self, _request: DecryptRequest) -> Result<DecryptResponse> {
unimplemented!("data plane is not audited by the manager")
}
async fn generate_data_key(&self, _request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> {
unimplemented!("data plane is not audited by the manager")
}
async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
self.check()?;
Ok(DescribeKeyResponse {
key_metadata: Self::metadata(&request.key_id),
})
}
async fn list_keys(&self, _request: ListKeysRequest) -> Result<ListKeysResponse> {
self.check()?;
Ok(ListKeysResponse {
keys: Vec::new(),
next_marker: None,
truncated: false,
})
}
async fn delete_key(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
self.check()?;
Ok(DeleteKeyResponse {
key_metadata: Self::metadata(&request.key_id),
key_id: request.key_id,
deletion_date: None,
})
}
async fn cancel_key_deletion(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> {
self.check()?;
Ok(CancelKeyDeletionResponse {
key_metadata: Self::metadata(&request.key_id),
key_id: request.key_id,
})
}
async fn enable_key(&self, _key_id: &str) -> Result<()> {
self.check()
}
async fn disable_key(&self, _key_id: &str) -> Result<()> {
self.check()
}
async fn rotate_key(&self, _key_id: &str) -> Result<()> {
self.check()
}
async fn health_check(&self) -> Result<bool> {
Ok(true)
}
}
const AUDITED_KEY_ID: &str = "audited-key";
/// Prefix length used when checking that a record embedded no *part* of a
/// secret. Long enough that a collision with unrelated text is not a real
/// concern.
const FRAGMENT_LEN: usize = 24;
fn scripted_manager(backend: ScriptedBackend) -> (KmsManager, Arc<CapturingSink>) {
let temp_dir = tempdir().expect("Failed to create temp dir");
let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
let sink = Arc::new(CapturingSink::default());
let manager = KmsManager::new(Arc::new(backend), config).with_audit_sink(sink.clone());
(manager, sink)
}
fn request_context() -> OperationContext {
OperationContext::new("arn:aws:iam::user/alice".to_string())
.with_source_ip("192.0.2.10".to_string())
.with_user_agent("rustfs-admin/1".to_string())
.with_context("requestID".to_string(), "req-42".to_string())
}
/// Drive one management operation and return the single record it emitted.
async fn run_operation(manager: &KmsManager, operation: KmsAuditOperation, context: &OperationContext) -> Result<()> {
match operation {
KmsAuditOperation::CreateKey => manager
.create_key_with_context(
CreateKeyRequest {
key_name: Some(AUDITED_KEY_ID.to_string()),
..Default::default()
},
context,
)
.await
.map(|_| ()),
KmsAuditOperation::DescribeKey => manager
.describe_key_with_context(
DescribeKeyRequest {
key_id: AUDITED_KEY_ID.to_string(),
},
context,
)
.await
.map(|_| ()),
KmsAuditOperation::ListKeys => manager
.list_keys_with_context(ListKeysRequest::default(), context)
.await
.map(|_| ()),
KmsAuditOperation::ScheduleKeyDeletion => manager
.delete_key_with_context(
DeleteKeyRequest {
key_id: AUDITED_KEY_ID.to_string(),
pending_window_in_days: None,
force_immediate: None,
},
context,
)
.await
.map(|_| ()),
KmsAuditOperation::CancelKeyDeletion => manager
.cancel_key_deletion_with_context(
CancelKeyDeletionRequest {
key_id: AUDITED_KEY_ID.to_string(),
},
context,
)
.await
.map(|_| ()),
KmsAuditOperation::EnableKey => manager.enable_key_with_context(AUDITED_KEY_ID, context).await,
KmsAuditOperation::DisableKey => manager.disable_key_with_context(AUDITED_KEY_ID, context).await,
KmsAuditOperation::RotateKey => manager.rotate_key_with_context(AUDITED_KEY_ID, context).await,
// Physical removal happens on the background sweep, not here.
KmsAuditOperation::DeleteKey => unreachable!("removal is audited by the deletion worker"),
}
}
/// Every management operation the manager serves, in audit terms.
const AUDITED_OPERATIONS: [KmsAuditOperation; 8] = [
KmsAuditOperation::CreateKey,
KmsAuditOperation::DescribeKey,
KmsAuditOperation::ListKeys,
KmsAuditOperation::ScheduleKeyDeletion,
KmsAuditOperation::CancelKeyDeletion,
KmsAuditOperation::EnableKey,
KmsAuditOperation::DisableKey,
KmsAuditOperation::RotateKey,
];
#[tokio::test]
async fn every_management_operation_emits_a_complete_success_record() {
for operation in AUDITED_OPERATIONS {
let (manager, sink) = scripted_manager(ScriptedBackend::succeeding());
let context = request_context();
let outer = Instant::now();
run_operation(&manager, operation, &context)
.await
.unwrap_or_else(|error| panic!("{} should succeed: {error}", operation.as_str()));
let outer_elapsed = outer.elapsed();
let record = sink.take_one();
assert_eq!(record.operation, operation);
assert_eq!(record.event, operation.event_name());
assert_eq!(record.outcome, KmsAuditOutcome::Success);
assert_eq!(record.error_class, None);
assert_eq!(record.operation_id, context.operation_id);
assert_eq!(record.principal, "arn:aws:iam::user/alice");
assert_eq!(record.source_ip.as_deref(), Some("192.0.2.10"));
assert_eq!(record.user_agent.as_deref(), Some("rustfs-admin/1"));
assert_eq!(record.backend, "local");
assert_eq!(record.context.get("requestID").map(String::as_str), Some("req-42"));
assert!(
record.latency <= outer_elapsed,
"{} reported a latency larger than the call it measured",
operation.as_str()
);
// Listing spans keys; every other operation names the key it touched.
if operation == KmsAuditOperation::ListKeys {
assert_eq!(record.key_id, None);
} else {
assert_eq!(record.key_id.as_deref(), Some(AUDITED_KEY_ID));
}
}
}
#[tokio::test]
async fn every_management_operation_emits_a_failure_record() {
for operation in AUDITED_OPERATIONS {
let (manager, sink) = scripted_manager(ScriptedBackend::failing(KmsError::access_denied("denied by policy")));
let context = request_context();
let error = run_operation(&manager, operation, &context)
.await
.expect_err("scripted backend should reject the operation");
assert!(matches!(error, KmsError::AccessDenied { .. }));
let record = sink.take_one();
assert_eq!(record.operation, operation);
assert_eq!(record.event, operation.event_name());
assert_eq!(record.outcome, KmsAuditOutcome::Failure);
assert_eq!(record.error_class, Some("access_denied"));
assert_eq!(record.operation_id, context.operation_id);
assert_eq!(record.principal, "arn:aws:iam::user/alice");
assert_eq!(record.source_ip.as_deref(), Some("192.0.2.10"));
// A denied create still has to name the key the caller asked for,
// otherwise the record cannot answer "what were they after".
if operation != KmsAuditOperation::ListKeys {
assert_eq!(record.key_id.as_deref(), Some(AUDITED_KEY_ID));
}
}
}
#[tokio::test]
async fn operations_are_unaffected_when_no_sink_is_installed() {
// The audit trail is optional; without a sink the manager must behave
// exactly as it did before records existed.
let temp_dir = tempdir().expect("Failed to create temp dir");
let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
let manager = KmsManager::new(Arc::new(ScriptedBackend::succeeding()), config);
for operation in AUDITED_OPERATIONS {
run_operation(&manager, operation, &request_context())
.await
.unwrap_or_else(|error| panic!("{} should succeed without a sink: {error}", operation.as_str()));
}
}
#[tokio::test]
async fn context_free_calls_are_attributed_to_the_internal_principal() {
// Callers that have no authenticated identity must still be
// distinguishable from an identity we failed to record.
let (manager, sink) = scripted_manager(ScriptedBackend::succeeding());
manager
.describe_key(DescribeKeyRequest {
key_id: AUDITED_KEY_ID.to_string(),
})
.await
.expect("describe should succeed");
let record = sink.take_one();
assert_eq!(record.principal, OperationContext::INTERNAL_PRINCIPAL);
assert_eq!(record.source_ip, None);
assert_eq!(record.user_agent, None);
}
#[tokio::test]
async fn unsupported_lifecycle_operations_are_audited_with_their_own_class() {
// The local backend has no version history, so rotation is a capability
// gap rather than a policy denial; the audit trail must say so.
let temp_dir = tempdir().expect("Failed to create temp dir");
let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
let sink = Arc::new(CapturingSink::default());
let manager = KmsManager::new(backend, config).with_audit_sink(sink.clone());
let key_id = manager
.create_key_with_context(
CreateKeyRequest {
key_name: Some("rotate-me".to_string()),
..Default::default()
},
&request_context(),
)
.await
.expect("create should succeed")
.key_id;
manager
.rotate_key_with_context(&key_id, &request_context())
.await
.expect_err("local rotation must be rejected");
let records = sink.records();
let rotate = records.last().expect("rotation should be audited");
assert_eq!(rotate.operation, KmsAuditOperation::RotateKey);
assert_eq!(rotate.outcome, KmsAuditOutcome::Failure);
assert_eq!(rotate.error_class, Some("unsupported_capability"));
assert_eq!(rotate.key_id.as_deref(), Some(key_id.as_str()));
}
/// Negative assertion: no audit record may reproduce key material. Driven
/// against the real local backend so the assertion covers whatever the
/// backend actually hands back, not a hand-written stand-in.
#[tokio::test]
async fn audit_records_never_reproduce_key_material() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
let sink = Arc::new(CapturingSink::default());
let manager = KmsManager::new(backend, config).with_audit_sink(sink.clone());
let grant_token = "grant-token-cec4d4b5a1";
let context = request_context();
let key_id = manager
.create_key_with_context(
CreateKeyRequest {
key_name: Some("material-key".to_string()),
..Default::default()
},
&context,
)
.await
.expect("create should succeed")
.key_id;
// Produce real key material, then keep driving the management plane so
// any record built afterwards is covered by the assertions below.
let data_key = manager
.generate_data_key(GenerateDataKeyRequest {
key_id: key_id.clone(),
key_spec: KeySpec::Aes256,
encryption_context: HashMap::from([("bucket".to_string(), "secrets".to_string())]),
})
.await
.expect("data key generation should succeed");
let decrypted = manager
.decrypt(DecryptRequest {
ciphertext: data_key.ciphertext_blob.clone(),
encryption_context: HashMap::from([("bucket".to_string(), "secrets".to_string())]),
grant_tokens: vec![grant_token.to_string()],
})
.await
.expect("decrypt should succeed");
manager
.describe_key_with_context(DescribeKeyRequest { key_id: key_id.clone() }, &context)
.await
.expect("describe should succeed");
manager
.list_keys_with_context(ListKeysRequest::default(), &context)
.await
.expect("list should succeed");
manager
.disable_key_with_context(&key_id, &context)
.await
.expect("disable should succeed");
manager
.enable_key_with_context(&key_id, &context)
.await
.expect("enable should succeed");
let base64 = base64::engine::general_purpose::STANDARD;
let encodings = |bytes: &[u8]| vec![hex::encode(bytes), base64.encode(bytes)];
let mut forbidden = vec![grant_token.to_string()];
forbidden.extend(encodings(&data_key.plaintext_key));
forbidden.extend(encodings(&decrypted.plaintext));
forbidden.extend(encodings(&data_key.ciphertext_blob));
// Fragments catch a record that embedded only part of a blob.
let fragments: Vec<String> = forbidden
.iter()
.filter(|secret| secret.len() > FRAGMENT_LEN)
.map(|secret| secret[..FRAGMENT_LEN].to_string())
.collect();
forbidden.extend(fragments);
let records = sink.records();
assert!(!records.is_empty(), "management operations should have been audited");
for record in &records {
let rendered = format!("{record:?}");
for secret in &forbidden {
assert!(
!rendered.contains(secret.as_str()),
"audit record leaked key material or a grant token: {rendered}"
);
}
}
}
#[tokio::test]
async fn test_manager_operations() {
let temp_dir = tempdir().expect("Failed to create temp dir");
@@ -904,12 +254,9 @@ mod tests {
let describe_response = manager.describe_key(describe_request).await.expect("Failed to describe key");
assert_eq!(describe_response.key_metadata.key_id, create_response.key_id);
// Creating the key populated the cache, so the describe above was
// served from it rather than from the backend.
let stats = manager.cache_stats().await.expect("cache is enabled");
assert_eq!(stats.entries, 1);
assert_eq!(stats.hits, 1);
assert_eq!(stats.misses, 0);
// Test cache stats
let stats = manager.cache_stats().await;
assert!(stats.is_some());
// Test health check
let health = manager.health_check().await.expect("Health check failed");
+1 -3
View File
@@ -104,9 +104,7 @@ pub(crate) fn classify_vaultrs(error: &vaultrs::error::ClientError) -> ErrorClas
}
}
/// Retry classification of an HTTP status, shared by every backend that talks
/// to an external KMS over HTTP.
pub(crate) fn classify_status(code: u16) -> ErrorClass {
fn classify_status(code: u16) -> ErrorClass {
match code {
429 | 500 | 502 | 503 | 504 => ErrorClass::RetryableStatus,
_ => ErrorClass::Fatal,
-894
View File
@@ -1,894 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Background synthetic probe for the configured KMS backend.
//!
//! A health check that only reports "the backend answered" cannot distinguish a
//! usable KMS from one that is reachable but no longer returns material that
//! round-trips. Each probe round therefore generates a data key under a
//! dedicated key, decrypts the returned ciphertext, and compares the plaintext,
//! so a backend that answers without being usable becomes visible before object
//! traffic depends on it.
//!
//! The probe key uses a fixed reserved id ([`PROBE_KEY_ID`]) and is created only
//! when missing, so every node of a deployment converges on the same key and a
//! create that loses the race against another node is a success, not an error.
//!
//! Backends that cannot host such a key — single-key read-only backends, or
//! backends without data key generation — are reported as
//! [`ProbeResult::Unsupported`] exactly once and then left alone: an
//! unsupported backend is a configuration fact, not an outage, and must never
//! raise failure metrics or alerts.
//!
//! Results are published two ways: as a lock-free [`ProbeStatus`] snapshot that
//! readiness reporting can read without touching the backend, and as
//! `rustfs_kms_probe_*` metrics. As everywhere else in this crate, metric labels
//! carry only static outcome classes — never key identifiers, key material, or
//! ciphertext.
use crate::backends::KmsBackend;
use crate::error::KmsError;
use crate::types::{CreateKeyRequest, DecryptRequest, DescribeKeyRequest, GenerateDataKeyRequest, KeySpec, KeyUsage};
use arc_swap::ArcSwap;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use subtle::ConstantTimeEq;
use tokio::time::Instant;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, warn};
use zeroize::Zeroize;
/// Reserved id of the key every probe round runs through.
///
/// Fixed rather than per-node or per-process: all nodes must converge on one
/// key instead of littering the backend with probe keys, and the reserved
/// `rustfs-internal-` prefix marks it as RustFS-owned for operators reviewing
/// the key list.
pub const PROBE_KEY_ID: &str = "rustfs-internal-kms-probe";
/// Description stored on the probe key when this node creates it.
const PROBE_KEY_DESCRIPTION: &str = "RustFS internal KMS health probe key; created and used only by the synthetic probe";
/// Default interval between probe rounds.
pub const DEFAULT_PROBE_INTERVAL: Duration = Duration::from_secs(60);
/// Floor for the configured interval. Probing faster buys no additional signal
/// and multiplies backend load by the size of the deployment.
pub const MIN_PROBE_INTERVAL: Duration = Duration::from_secs(5);
/// Probe interval in whole seconds; `0` disables the probe entirely.
pub const ENV_KMS_PROBE_INTERVAL_SECS: &str = "RUSTFS_KMS_PROBE_INTERVAL_SECS";
/// Result of the most recent probe round.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProbeResult {
/// No round has completed yet for this service version.
Pending,
/// The full generate/decrypt/compare cycle succeeded.
Success,
/// The backend cannot host the probe key; no further rounds run and no
/// failures are recorded.
Unsupported,
/// The round failed with the given classification.
Failure(ProbeFailureKind),
}
impl ProbeResult {
/// Static metric label for this result.
fn as_label(self) -> &'static str {
match self {
ProbeResult::Pending => "pending",
ProbeResult::Success => "success",
ProbeResult::Unsupported => "unsupported",
ProbeResult::Failure(_) => "failure",
}
}
}
/// Stage of the round trip that failed, used as a static metric label.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProbeFailureKind {
/// The dedicated probe key could not be described or created.
KeyProvisioning,
/// Data key generation failed.
Generate,
/// The freshly generated ciphertext could not be decrypted.
Decrypt,
/// Decryption succeeded but returned material that differs from what was
/// generated — the backend answers without round-tripping.
Mismatch,
}
impl ProbeFailureKind {
/// Static metric label for this failure class.
fn as_label(self) -> &'static str {
match self {
ProbeFailureKind::KeyProvisioning => "key_provisioning",
ProbeFailureKind::Generate => "generate",
ProbeFailureKind::Decrypt => "decrypt",
ProbeFailureKind::Mismatch => "mismatch",
}
}
}
/// Snapshot of what the probe has observed so far.
///
/// Consumed by readiness reporting, which owns the staleness and consecutive
/// failure thresholds; the probe itself only records what happened.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProbeStatus {
/// Result of the most recent completed round.
pub result: ProbeResult,
/// When the most recent round completed.
pub last_round_at: Option<Instant>,
/// When the most recent successful round completed.
pub last_success_at: Option<Instant>,
/// Wall-clock timestamp of the most recent success, in Unix seconds, for
/// display and for the exported gauge. Use [`ProbeStatus::last_success_age`]
/// for staleness decisions: it is monotonic and immune to clock steps.
pub last_success_unix_secs: Option<i64>,
/// Failed rounds since the last success; reset by any non-failure result.
pub consecutive_failures: u32,
}
impl ProbeStatus {
/// Initial snapshot published before the first round completes.
fn pending() -> Self {
Self {
result: ProbeResult::Pending,
last_round_at: None,
last_success_at: None,
last_success_unix_secs: None,
consecutive_failures: 0,
}
}
/// How long ago the most recent round completed.
pub fn last_round_age(&self) -> Option<Duration> {
self.last_round_at.map(|at| at.elapsed())
}
/// How long ago the most recent successful round completed.
pub fn last_success_age(&self) -> Option<Duration> {
self.last_success_at.map(|at| at.elapsed())
}
}
/// Lock-free publication slot for [`ProbeStatus`].
///
/// Written by the single worker task and read by anyone; readers never block
/// the probe and the probe never blocks a readiness handler.
struct ProbeState {
status: ArcSwap<ProbeStatus>,
}
impl ProbeState {
fn new() -> Self {
Self {
status: ArcSwap::from_pointee(ProbeStatus::pending()),
}
}
fn status(&self) -> Arc<ProbeStatus> {
self.status.load_full()
}
/// Fold one round result into the published snapshot.
///
/// Safe as a plain load-then-store because the owning worker task is the
/// only writer.
fn publish(&self, result: ProbeResult) {
let previous = self.status.load();
let now = Instant::now();
let (last_success_at, last_success_unix_secs) = match result {
ProbeResult::Success => (Some(now), Some(jiff::Timestamp::now().as_second())),
_ => (previous.last_success_at, previous.last_success_unix_secs),
};
let consecutive_failures = match result {
ProbeResult::Failure(_) => previous.consecutive_failures.saturating_add(1),
_ => 0,
};
self.status.store(Arc::new(ProbeStatus {
result,
last_round_at: Some(now),
last_success_at,
last_success_unix_secs,
consecutive_failures,
}));
record_state(result, last_success_unix_secs, consecutive_failures);
}
}
/// Owner of one service version's probe worker.
///
/// Mirrors the deletion worker's ownership model: cancelling stops the loop,
/// and dropping the handle — a service version replaced without an explicit
/// stop — cancels as a safety net.
pub struct ProbeHandle {
state: Arc<ProbeState>,
cancel: CancellationToken,
task: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
}
impl ProbeHandle {
/// Latest published snapshot.
pub fn status(&self) -> Arc<ProbeStatus> {
self.state.status()
}
/// Stop the worker. The task observes the cancelled token on its next poll.
pub(crate) fn shutdown(&self) {
self.cancel.cancel();
if let Ok(mut task) = self.task.lock() {
drop(task.take());
}
}
#[cfg(test)]
pub(crate) fn is_cancelled(&self) -> bool {
self.cancel.is_cancelled()
}
}
impl Drop for ProbeHandle {
fn drop(&mut self) {
self.cancel.cancel();
}
}
/// Start the probe for a service version.
///
/// Returns `None` when the backend advertises no data key round trip, or when
/// the operator disabled the probe; both cases leave no background task behind.
pub(crate) fn spawn_probe_worker(backend: Arc<dyn KmsBackend>) -> Option<ProbeHandle> {
let capabilities = backend.capabilities();
if !capabilities.generate_data_key || !capabilities.decrypt {
return None;
}
let interval = configured_interval()?;
let state = Arc::new(ProbeState::new());
let cancel = CancellationToken::new();
let task = ProbeWorker::new(backend, state.clone(), interval).spawn(cancel.clone());
Some(ProbeHandle {
state,
cancel,
task: std::sync::Mutex::new(Some(task)),
})
}
/// Probe interval from the environment, or `None` when the probe is disabled.
fn configured_interval() -> Option<Duration> {
parse_interval(std::env::var(ENV_KMS_PROBE_INTERVAL_SECS).ok().as_deref())
}
/// Interpret the configured interval: unset or unparsable falls back to the
/// default, `0` disables the probe, and anything below [`MIN_PROBE_INTERVAL`]
/// is raised to it so a misconfigured value cannot turn the probe into a load
/// generator against the backend.
fn parse_interval(value: Option<&str>) -> Option<Duration> {
let Some(value) = value else {
return Some(DEFAULT_PROBE_INTERVAL);
};
let Ok(seconds) = value.trim().parse::<u64>() else {
warn!(
variable = ENV_KMS_PROBE_INTERVAL_SECS,
"ignoring unparsable KMS probe interval; falling back to the default"
);
return Some(DEFAULT_PROBE_INTERVAL);
};
if seconds == 0 {
return None;
}
Some(Duration::from_secs(seconds).max(MIN_PROBE_INTERVAL))
}
/// Encryption context bound to every probe data key.
///
/// Static and free of per-node values, so material generated by one node stays
/// decryptable by every other node running the same probe.
fn probe_encryption_context() -> HashMap<String, String> {
HashMap::from([("rustfs-purpose".to_string(), "kms-probe".to_string())])
}
/// Outcome of making sure the dedicated probe key exists.
enum KeyProvisioning {
Ready,
Unsupported,
Failed,
}
pub(crate) struct ProbeWorker {
backend: Arc<dyn KmsBackend>,
state: Arc<ProbeState>,
interval: Duration,
}
impl ProbeWorker {
fn new(backend: Arc<dyn KmsBackend>, state: Arc<ProbeState>, interval: Duration) -> Self {
Self {
backend,
state,
interval,
}
}
fn spawn(self, cancel: CancellationToken) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move { self.run(cancel).await })
}
async fn run(self, cancel: CancellationToken) {
describe_metrics();
let mut ticker = tokio::time::interval(self.interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
_ = cancel.cancelled() => {
debug!("KMS probe worker stopped");
return;
}
_ = ticker.tick() => {}
}
let started = Instant::now();
let result = self.round().await;
record_round(result, started.elapsed());
self.state.publish(result);
if result == ProbeResult::Unsupported {
// A capability gap does not change while a service version
// lives, so retrying it forever would only produce noise.
info!("KMS backend cannot host the synthetic probe key; probe disabled for this service version");
return;
}
}
}
/// Run one round. Exposed separately so tests can drive the round trip
/// deterministically without the interval loop.
pub(crate) async fn round(&self) -> ProbeResult {
let capabilities = self.backend.capabilities();
if !capabilities.generate_data_key || !capabilities.decrypt {
return ProbeResult::Unsupported;
}
match self.ensure_probe_key().await {
KeyProvisioning::Ready => {}
KeyProvisioning::Unsupported => return ProbeResult::Unsupported,
KeyProvisioning::Failed => return ProbeResult::Failure(ProbeFailureKind::KeyProvisioning),
}
let mut generated = match self
.backend
.generate_data_key(GenerateDataKeyRequest {
key_id: PROBE_KEY_ID.to_string(),
key_spec: KeySpec::Aes256,
encryption_context: probe_encryption_context(),
})
.await
{
Ok(generated) => generated,
Err(error) => {
warn!(%error, "KMS probe could not generate a data key");
return ProbeResult::Failure(ProbeFailureKind::Generate);
}
};
let decrypted = self
.backend
.decrypt(DecryptRequest {
ciphertext: generated.ciphertext_blob.clone(),
encryption_context: probe_encryption_context(),
grant_tokens: Vec::new(),
})
.await;
let result = match decrypted {
Ok(mut response) => {
let round_trips = bool::from(response.plaintext.ct_eq(&generated.plaintext_key));
response.plaintext.zeroize();
if round_trips {
debug!("KMS probe round trip succeeded");
ProbeResult::Success
} else {
// The backend answered both calls but the material does not
// survive the round trip: report it as loudly as an outage.
warn!("KMS probe decrypted material does not match the generated data key");
ProbeResult::Failure(ProbeFailureKind::Mismatch)
}
}
Err(error) => {
warn!(%error, "KMS probe could not decrypt its generated data key");
ProbeResult::Failure(ProbeFailureKind::Decrypt)
}
};
generated.plaintext_key.zeroize();
result
}
/// Create the probe key if it is missing.
///
/// Idempotent by construction: an existing key is used as is, and a create
/// that loses the race against another node reports the key as ready
/// instead of failing the round.
async fn ensure_probe_key(&self) -> KeyProvisioning {
match self
.backend
.describe_key(DescribeKeyRequest {
key_id: PROBE_KEY_ID.to_string(),
})
.await
{
Ok(_) => return KeyProvisioning::Ready,
Err(KmsError::KeyNotFound { .. }) => {}
Err(error) if is_capability_gap(&error) => return KeyProvisioning::Unsupported,
Err(error) => {
warn!(%error, "KMS probe could not describe its probe key");
return KeyProvisioning::Failed;
}
}
match self
.backend
.create_key(CreateKeyRequest {
key_name: Some(PROBE_KEY_ID.to_string()),
key_usage: KeyUsage::EncryptDecrypt,
description: Some(PROBE_KEY_DESCRIPTION.to_string()),
..Default::default()
})
.await
{
Ok(_) => {
info!(key_id = PROBE_KEY_ID, "created the KMS synthetic probe key");
KeyProvisioning::Ready
}
// Another node created the key between the describe and the create.
Err(KmsError::KeyAlreadyExists { .. }) => KeyProvisioning::Ready,
Err(error) if is_capability_gap(&error) => KeyProvisioning::Unsupported,
Err(error) => {
warn!(%error, "KMS probe could not create its probe key");
KeyProvisioning::Failed
}
}
}
}
/// Whether an error means "this backend will never host a probe key".
///
/// Read-only single-key backends reject key creation outright, and backends
/// without the operation report an unsupported capability. Both are properties
/// of the configuration rather than symptoms of an outage, so they stop the
/// probe instead of counting as failures.
fn is_capability_gap(error: &KmsError) -> bool {
matches!(error, KmsError::UnsupportedCapability { .. } | KmsError::InvalidOperation { .. })
}
// ---------------------------------------------------------------------------
// Metrics
//
// Label values are exclusively static outcome classes; the probe key id is a
// compile-time constant and still never becomes a label, so the metric shape
// stays independent of how many keys or nodes exist.
// ---------------------------------------------------------------------------
/// Counter: probe rounds completed, by `result`.
const METRIC_PROBE_ROUNDS_TOTAL: &str = "rustfs_kms_probe_rounds_total";
/// Counter: failed probe rounds, by `failure_kind`.
const METRIC_PROBE_FAILURES_TOTAL: &str = "rustfs_kms_probe_failures_total";
/// Histogram: wall-clock duration of one probe round, in seconds, by `result`.
const METRIC_PROBE_DURATION_SECONDS: &str = "rustfs_kms_probe_duration_seconds";
/// Gauge: Unix timestamp of the most recent successful round.
const METRIC_PROBE_LAST_SUCCESS_TIMESTAMP: &str = "rustfs_kms_probe_last_success_timestamp_seconds";
/// Gauge: failed rounds since the last success.
const METRIC_PROBE_CONSECUTIVE_FAILURES: &str = "rustfs_kms_probe_consecutive_failures";
/// Register metric descriptions once per process.
fn describe_metrics() {
static DESCRIBE: std::sync::Once = std::sync::Once::new();
DESCRIBE.call_once(|| {
metrics::describe_counter!(METRIC_PROBE_ROUNDS_TOTAL, "Total synthetic KMS probe rounds completed, by result");
metrics::describe_counter!(
METRIC_PROBE_FAILURES_TOTAL,
"Total failed synthetic KMS probe rounds, by the round-trip stage that failed"
);
metrics::describe_histogram!(
METRIC_PROBE_DURATION_SECONDS,
"Wall-clock duration of one synthetic KMS probe round, in seconds"
);
metrics::describe_gauge!(
METRIC_PROBE_LAST_SUCCESS_TIMESTAMP,
"Unix timestamp of the most recent successful synthetic KMS probe round"
);
metrics::describe_gauge!(
METRIC_PROBE_CONSECUTIVE_FAILURES,
"Synthetic KMS probe rounds that have failed since the last success"
);
});
}
/// Record one completed round.
///
/// An unsupported backend is deliberately counted as its own result and never
/// as a failure, so alerts on the failure counter stay silent for deployments
/// whose backend cannot host the probe.
fn record_round(result: ProbeResult, elapsed: Duration) {
metrics::counter!(METRIC_PROBE_ROUNDS_TOTAL, "result" => result.as_label()).increment(1);
if let ProbeResult::Failure(kind) = result {
metrics::counter!(METRIC_PROBE_FAILURES_TOTAL, "failure_kind" => kind.as_label()).increment(1);
}
metrics::histogram!(METRIC_PROBE_DURATION_SECONDS, "result" => result.as_label()).record(elapsed.as_secs_f64());
}
/// Mirror the published snapshot into the gauges.
fn record_state(result: ProbeResult, last_success_unix_secs: Option<i64>, consecutive_failures: u32) {
if result == ProbeResult::Success
&& let Some(timestamp) = last_success_unix_secs
{
// Only ever moved forward by a success, so a failing probe leaves the
// gauge at the last known good time and its age keeps growing.
metrics::gauge!(METRIC_PROBE_LAST_SUCCESS_TIMESTAMP).set(timestamp as f64);
}
metrics::gauge!(METRIC_PROBE_CONSECUTIVE_FAILURES).set(f64::from(consecutive_failures));
}
#[cfg(test)]
mod tests {
use super::*;
use crate::backends::local::LocalKmsBackend;
use crate::backends::static_kms::StaticKmsBackend;
use crate::config::KmsConfig;
use crate::types::{
CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyResponse, DecryptResponse, DeleteKeyRequest,
DeleteKeyResponse, DescribeKeyResponse, EncryptRequest, EncryptResponse, GenerateDataKeyResponse, KeyMetadata, KeyState,
ListKeysRequest, ListKeysResponse,
};
use async_trait::async_trait;
use base64::Engine as _;
use metrics_util::MetricKind;
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
use std::future::Future;
async fn local_backend(temp_dir: &tempfile::TempDir) -> Arc<dyn KmsBackend> {
let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
Arc::new(LocalKmsBackend::new(config).await.expect("local backend should build"))
}
async fn static_backend() -> Arc<dyn KmsBackend> {
let config =
KmsConfig::static_kms("static-key".to_string(), base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]));
Arc::new(StaticKmsBackend::new(config).await.expect("static backend should build"))
}
fn worker(backend: Arc<dyn KmsBackend>) -> ProbeWorker {
ProbeWorker::new(backend, Arc::new(ProbeState::new()), DEFAULT_PROBE_INTERVAL)
}
/// Backend that answers every probe call but corrupts the round trip in a
/// configurable way, so each failure classification can be provoked without
/// a real backend outage.
struct BrokenBackend {
fault: Fault,
}
#[derive(Clone, Copy)]
enum Fault {
Generate,
Decrypt,
/// Decrypt succeeds but returns material other than the generated key.
Mismatch,
}
impl BrokenBackend {
fn arc(fault: Fault) -> Arc<dyn KmsBackend> {
Arc::new(Self { fault })
}
fn probe_key_metadata() -> KeyMetadata {
KeyMetadata {
key_id: PROBE_KEY_ID.to_string(),
key_state: KeyState::Enabled,
key_usage: KeyUsage::EncryptDecrypt,
description: None,
creation_date: jiff::Zoned::now(),
deletion_date: None,
origin: "test".to_string(),
key_manager: "test".to_string(),
tags: HashMap::new(),
}
}
}
#[async_trait]
impl KmsBackend for BrokenBackend {
async fn create_key(&self, _request: CreateKeyRequest) -> crate::error::Result<CreateKeyResponse> {
unimplemented!("the probe key always exists on this backend")
}
async fn encrypt(&self, _request: EncryptRequest) -> crate::error::Result<EncryptResponse> {
unimplemented!("not exercised by probe tests")
}
async fn decrypt(&self, _request: DecryptRequest) -> crate::error::Result<DecryptResponse> {
match self.fault {
Fault::Decrypt => Err(KmsError::backend_error("decrypt unavailable")),
Fault::Mismatch => Ok(DecryptResponse {
plaintext: vec![0xffu8; 32],
key_id: PROBE_KEY_ID.to_string(),
encryption_algorithm: None,
}),
Fault::Generate => unreachable!("generation fails before decrypt is reached"),
}
}
async fn generate_data_key(&self, _request: GenerateDataKeyRequest) -> crate::error::Result<GenerateDataKeyResponse> {
match self.fault {
Fault::Generate => Err(KmsError::backend_error("data key generation unavailable")),
_ => Ok(GenerateDataKeyResponse {
key_id: PROBE_KEY_ID.to_string(),
plaintext_key: vec![0x11u8; 32],
ciphertext_blob: vec![0x22u8; 48],
}),
}
}
async fn describe_key(&self, _request: DescribeKeyRequest) -> crate::error::Result<DescribeKeyResponse> {
Ok(DescribeKeyResponse {
key_metadata: Self::probe_key_metadata(),
})
}
async fn list_keys(&self, _request: ListKeysRequest) -> crate::error::Result<ListKeysResponse> {
unimplemented!("not exercised by probe tests")
}
async fn delete_key(&self, _request: DeleteKeyRequest) -> crate::error::Result<DeleteKeyResponse> {
unimplemented!("not exercised by probe tests")
}
async fn cancel_key_deletion(
&self,
_request: CancelKeyDeletionRequest,
) -> crate::error::Result<CancelKeyDeletionResponse> {
unimplemented!("not exercised by probe tests")
}
async fn health_check(&self) -> crate::error::Result<bool> {
Ok(true)
}
}
#[tokio::test]
async fn round_creates_the_probe_key_once_and_round_trips_it() {
let temp_dir = tempfile::tempdir().expect("temp dir");
let backend = local_backend(&temp_dir).await;
let worker = worker(backend.clone());
assert_eq!(worker.round().await, ProbeResult::Success);
let created = backend
.describe_key(DescribeKeyRequest {
key_id: PROBE_KEY_ID.to_string(),
})
.await
.expect("the first round must create the probe key");
assert_eq!(created.key_metadata.key_id, PROBE_KEY_ID);
// A second round reuses the key instead of failing on the existing one,
// which is also what a second node in the deployment would do.
assert_eq!(worker.round().await, ProbeResult::Success);
assert_eq!(
backend
.list_keys(ListKeysRequest {
limit: Some(100),
marker: None,
usage_filter: None,
status_filter: None,
})
.await
.expect("list keys")
.keys
.iter()
.filter(|key| key.key_id == PROBE_KEY_ID)
.count(),
1,
"probe key creation must be idempotent"
);
}
#[test]
fn read_only_backend_is_reported_unsupported_without_failures() {
let (snapshot, result) = record_metrics(|| {
Box::pin(async {
let worker = worker(static_backend().await);
let result = worker.round().await;
record_round(result, Duration::from_millis(1));
result
})
});
assert_eq!(result, ProbeResult::Unsupported);
assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "unsupported")]), 1);
assert_eq!(
counter_value(&snapshot, METRIC_PROBE_FAILURES_TOTAL, &[]),
0,
"an unsupported backend must never raise probe failures"
);
}
#[tokio::test]
async fn each_round_trip_stage_gets_its_own_failure_classification() {
for (fault, expected) in [
(Fault::Generate, ProbeFailureKind::Generate),
(Fault::Decrypt, ProbeFailureKind::Decrypt),
(Fault::Mismatch, ProbeFailureKind::Mismatch),
] {
assert_eq!(worker(BrokenBackend::arc(fault)).round().await, ProbeResult::Failure(expected));
}
}
#[test]
fn metrics_and_status_record_success_then_failure() {
let (snapshot, status) = record_metrics(|| {
Box::pin(async {
let temp_dir = tempfile::tempdir().expect("temp dir");
let state = Arc::new(ProbeState::new());
let healthy = ProbeWorker::new(local_backend(&temp_dir).await, state.clone(), DEFAULT_PROBE_INTERVAL);
let round = healthy.round().await;
record_round(round, Duration::from_millis(10));
state.publish(round);
let broken = ProbeWorker::new(BrokenBackend::arc(Fault::Mismatch), state.clone(), DEFAULT_PROBE_INTERVAL);
for _ in 0..2 {
let round = broken.round().await;
record_round(round, Duration::from_millis(10));
state.publish(round);
}
state.status()
})
});
assert_eq!(status.result, ProbeResult::Failure(ProbeFailureKind::Mismatch));
assert_eq!(status.consecutive_failures, 2);
assert!(
status.last_success_at.is_some() && status.last_success_unix_secs.is_some(),
"a failing probe must keep reporting when it last succeeded"
);
assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "success")]), 1);
assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "failure")]), 2);
assert_eq!(counter_value(&snapshot, METRIC_PROBE_FAILURES_TOTAL, &[("failure_kind", "mismatch")]), 2);
assert_eq!(histogram_values(&snapshot, METRIC_PROBE_DURATION_SECONDS, &[]).len(), 3);
assert!(
gauge_value(&snapshot, METRIC_PROBE_LAST_SUCCESS_TIMESTAMP).is_some_and(|timestamp| timestamp > 0.0),
"the last success gauge must carry a wall-clock timestamp"
);
assert_eq!(gauge_value(&snapshot, METRIC_PROBE_CONSECUTIVE_FAILURES), Some(2.0));
}
#[tokio::test(start_paused = true)]
async fn worker_publishes_status_and_stops_on_cancel() {
let temp_dir = tempfile::tempdir().expect("temp dir");
let state = Arc::new(ProbeState::new());
assert_eq!(state.status().result, ProbeResult::Pending);
let cancel = CancellationToken::new();
let task = ProbeWorker::new(local_backend(&temp_dir).await, state.clone(), DEFAULT_PROBE_INTERVAL).spawn(cancel.clone());
// The paused clock auto-advances through the worker's interval ticks.
let mut observed = None;
for _ in 0..100 {
tokio::time::sleep(Duration::from_secs(1)).await;
if state.status().result != ProbeResult::Pending {
observed = Some(state.status());
break;
}
}
let observed = observed.expect("the worker loop must publish a round result");
assert_eq!(observed.result, ProbeResult::Success);
assert_eq!(observed.consecutive_failures, 0);
assert!(observed.last_success_at.is_some());
cancel.cancel();
task.await.expect("worker task must stop after cancellation");
}
#[tokio::test(start_paused = true)]
async fn unsupported_backend_stops_the_worker_without_cancellation() {
let state = Arc::new(ProbeState::new());
let cancel = CancellationToken::new();
let task = ProbeWorker::new(static_backend().await, state.clone(), DEFAULT_PROBE_INTERVAL).spawn(cancel.clone());
task.await.expect("worker must exit on its own for an unsupported backend");
assert_eq!(state.status().result, ProbeResult::Unsupported);
assert!(!cancel.is_cancelled());
}
#[test]
fn interval_configuration_defaults_clamps_and_disables() {
assert_eq!(parse_interval(None), Some(DEFAULT_PROBE_INTERVAL));
assert_eq!(parse_interval(Some("not-a-number")), Some(DEFAULT_PROBE_INTERVAL));
assert_eq!(parse_interval(Some(" 120 ")), Some(Duration::from_secs(120)));
assert_eq!(parse_interval(Some("1")), Some(MIN_PROBE_INTERVAL));
assert_eq!(parse_interval(Some("0")), None, "zero must disable the probe");
}
// -- Metric assertion helpers, mirroring the policy engine's tests --------
type MetricEntry = (
metrics_util::CompositeKey,
Option<metrics::Unit>,
Option<metrics::SharedString>,
DebugValue,
);
/// Run `test` on a paused current-thread runtime under a debugging recorder
/// and return one snapshot of everything it emitted.
fn record_metrics<Out>(test: impl FnOnce() -> std::pin::Pin<Box<dyn Future<Output = Out>>>) -> (Vec<MetricEntry>, Out) {
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let out = metrics::with_local_recorder(&recorder, || {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_time()
.start_paused(true)
.build()
.expect("current-thread runtime must build");
runtime.block_on(test())
});
(snapshotter.snapshot().into_vec(), out)
}
fn labels_match(key: &metrics::Key, labels: &[(&str, &str)]) -> bool {
labels
.iter()
.all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected))
}
fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 {
snapshot
.iter()
.filter_map(|(composite, _unit, _description, value)| {
let matches = composite.kind() == MetricKind::Counter
&& composite.key().name() == name
&& labels_match(composite.key(), labels);
match (matches, value) {
(true, DebugValue::Counter(count)) => Some(*count),
_ => None,
}
})
.sum()
}
fn histogram_values(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> Vec<f64> {
snapshot
.iter()
.filter_map(|(composite, _unit, _description, value)| {
let matches = composite.kind() == MetricKind::Histogram
&& composite.key().name() == name
&& labels_match(composite.key(), labels);
match (matches, value) {
(true, DebugValue::Histogram(values)) => Some(values),
_ => None,
}
})
.flatten()
.map(|value| value.into_inner())
.collect()
}
fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
snapshot.iter().find_map(|(composite, _unit, _description, value)| {
let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
match (matches, value) {
(true, DebugValue::Gauge(gauge)) => Some(gauge.into_inner()),
_ => None,
}
})
}
}
+2 -233
View File
@@ -14,10 +14,6 @@
//! Object encryption service for S3-compatible encryption
use crate::api_types::{
TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse, UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
};
use crate::cache::KmsCacheStats;
use crate::encryption::ciphers::{create_cipher, generate_iv};
use crate::error::{KmsError, Result};
use crate::manager::KmsManager;
@@ -116,23 +112,6 @@ impl ObjectEncryptionService {
self.kms_manager.create_key(request).await
}
/// Create a new master key on behalf of `context`'s principal
///
/// # Arguments
/// * `request` - CreateKeyRequest with key parameters
/// * `context` - Identity and correlation data recorded in the audit trail
///
/// # Returns
/// CreateKeyResponse with created key details
///
pub async fn create_key_with_context(
&self,
request: CreateKeyRequest,
context: &OperationContext,
) -> Result<CreateKeyResponse> {
self.kms_manager.create_key_with_context(request, context).await
}
/// Describe a master key (delegates to KMS manager)
///
/// # Arguments
@@ -145,23 +124,6 @@ impl ObjectEncryptionService {
self.kms_manager.describe_key(request).await
}
/// Describe a master key on behalf of `context`'s principal
///
/// # Arguments
/// * `request` - DescribeKeyRequest with key ID
/// * `context` - Identity and correlation data recorded in the audit trail
///
/// # Returns
/// DescribeKeyResponse with key metadata
///
pub async fn describe_key_with_context(
&self,
request: DescribeKeyRequest,
context: &OperationContext,
) -> Result<DescribeKeyResponse> {
self.kms_manager.describe_key_with_context(request, context).await
}
/// List master keys (delegates to KMS manager)
///
/// # Arguments
@@ -174,77 +136,6 @@ impl ObjectEncryptionService {
self.kms_manager.list_keys(request).await
}
/// List master keys on behalf of `context`'s principal
///
/// # Arguments
/// * `request` - ListKeysRequest with listing parameters
/// * `context` - Identity and correlation data recorded in the audit trail
///
/// # Returns
/// ListKeysResponse with list of keys
///
pub async fn list_keys_with_context(&self, request: ListKeysRequest, context: &OperationContext) -> Result<ListKeysResponse> {
self.kms_manager.list_keys_with_context(request, context).await
}
/// Replace a master key's description (delegates to KMS manager)
///
/// # Arguments
/// * `request` - UpdateKeyDescriptionRequest with key ID and the new
/// description; an empty description clears the stored value
///
/// # Returns
/// UpdateKeyDescriptionResponse acknowledging the update
///
pub async fn update_key_description(&self, request: UpdateKeyDescriptionRequest) -> Result<UpdateKeyDescriptionResponse> {
let description = (!request.description.is_empty()).then_some(request.description.as_str());
self.kms_manager.update_key_description(&request.key_id, description).await?;
Ok(UpdateKeyDescriptionResponse {
success: true,
message: "key description updated".to_string(),
key_id: request.key_id,
})
}
/// Add or overwrite master key tags (delegates to KMS manager)
///
/// # Arguments
/// * `request` - TagKeyRequest with key ID and the tags to set; tags
/// outside the request are left untouched
///
/// # Returns
/// TagKeyResponse acknowledging the update
///
pub async fn tag_key(&self, request: TagKeyRequest) -> Result<TagKeyResponse> {
self.kms_manager.tag_key(&request.key_id, &request.tags).await?;
Ok(TagKeyResponse {
success: true,
message: "key tags updated".to_string(),
key_id: request.key_id,
})
}
/// Remove master key tags (delegates to KMS manager)
///
/// # Arguments
/// * `request` - UntagKeyRequest with key ID and the tag keys to remove;
/// tag keys that are not set are ignored, so the call is idempotent
///
/// # Returns
/// UntagKeyResponse acknowledging the removal
///
pub async fn untag_key(&self, request: UntagKeyRequest) -> Result<UntagKeyResponse> {
self.kms_manager.untag_key(&request.key_id, &request.tag_keys).await?;
Ok(UntagKeyResponse {
success: true,
message: "key tags removed".to_string(),
key_id: request.key_id,
})
}
/// Generate a data encryption key (delegates to KMS manager)
///
/// # Arguments
@@ -269,9 +160,9 @@ impl ObjectEncryptionService {
/// Get cache statistics
///
/// # Returns
/// A [`KmsCacheStats`] snapshot if caching is enabled, `None` otherwise
/// Option with (hits, misses) if caching is enabled
///
pub async fn cache_stats(&self) -> Option<KmsCacheStats> {
pub async fn cache_stats(&self) -> Option<(u64, u64)> {
self.kms_manager.cache_stats().await
}
@@ -1027,128 +918,6 @@ mod tests {
);
}
async fn describe(service: &ObjectEncryptionService, key_id: &str) -> KeyMetadata {
service
.describe_key(DescribeKeyRequest {
key_id: key_id.to_string(),
})
.await
.expect("describe should succeed")
.key_metadata
}
async fn create_metadata_test_key(service: &ObjectEncryptionService, key_id: &str) {
service
.create_key(CreateKeyRequest {
key_name: Some(key_id.to_string()),
key_usage: KeyUsage::EncryptDecrypt,
description: Some("original".to_string()),
policy: None,
tags: HashMap::from([
("name".to_string(), key_id.to_string()),
("team".to_string(), "storage".to_string()),
]),
origin: None,
})
.await
.expect("test key should be created");
}
#[tokio::test]
async fn key_metadata_updates_are_visible_to_describe() {
let (service, _temp_dir) = create_test_service().await;
create_metadata_test_key(&service, "metadata-key").await;
service
.update_key_description(UpdateKeyDescriptionRequest {
key_id: "metadata-key".to_string(),
description: "updated".to_string(),
})
.await
.expect("description update should succeed");
service
.tag_key(TagKeyRequest {
key_id: "metadata-key".to_string(),
tags: HashMap::from([
("team".to_string(), "platform".to_string()),
("env".to_string(), "prod".to_string()),
]),
})
.await
.expect("tagging should succeed");
// Reading through the manager's metadata cache must observe the
// updates, not the snapshot cached when the key was created.
let metadata = describe(&service, "metadata-key").await;
assert_eq!(metadata.description.as_deref(), Some("updated"));
assert_eq!(metadata.tags.get("team").map(String::as_str), Some("platform"));
assert_eq!(metadata.tags.get("env").map(String::as_str), Some("prod"));
assert_eq!(
metadata.tags.get("name").map(String::as_str),
Some("metadata-key"),
"tags set at creation must survive a later tag update"
);
// Untagging is idempotent: removing an absent tag is a no-op, not an
// error, so a retried request stays safe.
for attempt in 1..=2 {
service
.untag_key(UntagKeyRequest {
key_id: "metadata-key".to_string(),
tag_keys: vec!["env".to_string()],
})
.await
.unwrap_or_else(|error| panic!("untag attempt {attempt} should succeed: {error:?}"));
}
let metadata = describe(&service, "metadata-key").await;
assert!(!metadata.tags.contains_key("env"), "untagged tag must be gone: {:?}", metadata.tags);
assert_eq!(
metadata.tags.get("team").map(String::as_str),
Some("platform"),
"untagging must not touch other tags"
);
// An empty description clears the stored value.
service
.update_key_description(UpdateKeyDescriptionRequest {
key_id: "metadata-key".to_string(),
description: String::new(),
})
.await
.expect("clearing the description should succeed");
assert_eq!(describe(&service, "metadata-key").await.description, None);
}
#[tokio::test]
async fn identity_tag_is_not_writable_through_metadata_updates() {
let (service, _temp_dir) = create_test_service().await;
create_metadata_test_key(&service, "identity-key").await;
let rewrite = service
.tag_key(TagKeyRequest {
key_id: "identity-key".to_string(),
tags: HashMap::from([("name".to_string(), "other-key".to_string())]),
})
.await
.expect_err("rewriting the identity tag must be rejected");
assert!(matches!(rewrite, KmsError::InvalidOperation { .. }), "got {rewrite:?}");
let removal = service
.untag_key(UntagKeyRequest {
key_id: "identity-key".to_string(),
tag_keys: vec!["team".to_string(), "name".to_string()],
})
.await
.expect_err("removing the identity tag must be rejected");
assert!(matches!(removal, KmsError::InvalidOperation { .. }), "got {removal:?}");
// Both rejections are pre-write: the record is untouched, including
// the ordinary tag that shared the rejected untag request.
let metadata = describe(&service, "identity-key").await;
assert_eq!(metadata.tags.get("name").map(String::as_str), Some("identity-key"));
assert_eq!(metadata.tags.get("team").map(String::as_str), Some("storage"));
}
#[tokio::test]
async fn test_decrypt_data_key_uses_object_encryption_context() {
let (service, _temp_dir) = create_test_service().await;
+5 -112
View File
@@ -14,14 +14,12 @@
//! KMS service manager for dynamic configuration and runtime management
use crate::audit::KmsAuditSink;
use crate::backends::vault_credentials::CredentialTaskHandle;
use crate::backends::{KmsBackend, local::LocalKmsBackend};
use crate::config::{BackendConfig, KmsConfig};
use crate::deletion_worker::{DeletionReferenceChecker, DeletionWorker};
use crate::error::{KmsError, Result};
use crate::manager::KmsManager;
use crate::probe::{ProbeHandle, ProbeStatus, spawn_probe_worker};
use crate::service::ObjectEncryptionService;
use arc_swap::ArcSwap;
use sha2::{Digest, Sha256};
@@ -122,23 +120,13 @@ struct ServiceVersion {
/// Background deletion worker owned by this service version, if the
/// backend supports deletion scheduling
deletion_worker: Option<Arc<DeletionWorkerHandle>>,
/// Background synthetic probe owned by this service version, if the
/// backend can round-trip a data key and the probe is enabled
probe_worker: Option<Arc<ProbeHandle>>,
}
impl ServiceVersion {
/// Stop the background workers this version owns.
///
/// The credential renewal task is recycled separately because its shutdown
/// is asynchronous.
fn shutdown_background_workers(&self) {
fn shutdown_deletion_worker(&self) {
if let Some(worker) = &self.deletion_worker {
worker.shutdown();
}
if let Some(probe) = &self.probe_worker {
probe.shutdown();
}
}
}
@@ -183,8 +171,6 @@ pub struct KmsServiceManager {
lifecycle_mutex: Arc<Mutex<()>>,
/// External reference checker consulted before expired keys are removed
deletion_reference_checker: std::sync::RwLock<Option<Arc<dyn DeletionReferenceChecker>>>,
/// External sink receiving audit records for management operations
audit_sink: std::sync::RwLock<Option<Arc<dyn KmsAuditSink>>>,
}
impl KmsServiceManager {
@@ -199,26 +185,9 @@ impl KmsServiceManager {
version_counter: Arc::new(AtomicU64::new(0)),
lifecycle_mutex: Arc::new(Mutex::new(())),
deletion_reference_checker: std::sync::RwLock::new(None),
audit_sink: std::sync::RwLock::new(None),
}
}
/// Install the sink that receives an audit record for every KMS
/// management operation. Takes effect for services built by the next
/// start or reconfigure.
///
/// Without a sink no records are built, so KMS operations are unaffected
/// when auditing is not configured.
pub fn set_audit_sink(&self, sink: Arc<dyn KmsAuditSink>) {
if let Ok(mut slot) = self.audit_sink.write() {
*slot = Some(sink);
}
}
fn audit_sink(&self) -> Option<Arc<dyn KmsAuditSink>> {
self.audit_sink.read().ok().and_then(|slot| slot.clone())
}
/// Install the reference checker consulted before the deletion worker
/// removes an expired key. Takes effect for workers spawned by the next
/// start or reconfigure.
@@ -410,7 +379,7 @@ impl KmsServiceManager {
// Note: Existing Arc references will keep the service alive until operations complete
let state = self.state.load_full();
if let Some(current) = state.current_service.as_ref() {
current.shutdown_background_workers();
current.shutdown_deletion_worker();
}
self.state.store(Arc::new(RuntimeState {
config: state.config.clone(),
@@ -545,19 +514,6 @@ impl KmsServiceManager {
self.state.load().current_service.as_ref().map(|sv| sv.version)
}
/// Latest synthetic probe snapshot of the running service version.
///
/// `None` when KMS is not running, when the backend cannot round-trip a
/// data key, or when the probe is disabled. The read is lock-free and
/// performs no backend I/O, so a health endpoint can consult it without
/// turning an external KMS hiccup into probe pressure of its own; the
/// staleness and consecutive-failure thresholds belong to the caller.
pub fn probe_status(&self) -> Option<Arc<ProbeStatus>> {
let state = self.state.load();
let service_version = state.current_service.as_ref()?;
Some(service_version.probe_worker.as_ref()?.status())
}
/// Health check for the KMS service
pub async fn health_check(&self) -> Result<bool> {
let checked_state = self.state.load_full();
@@ -626,19 +582,10 @@ impl KmsServiceManager {
let backend = crate::backends::static_kms::StaticKmsBackend::new(config.clone()).await?;
Arc::new(backend) as Arc<dyn KmsBackend>
}
BackendConfig::Aws(_) => {
info!("Creating AWS KMS backend for version {}", version);
let backend = crate::backends::aws::AwsKmsBackend::new(config.clone()).await?;
Arc::new(backend) as Arc<dyn KmsBackend>
}
};
// Create KMS manager
let mut kms_manager = KmsManager::new(backend, config.clone());
if let Some(sink) = self.audit_sink() {
kms_manager = kms_manager.with_audit_sink(sink);
}
let kms_manager = Arc::new(kms_manager);
let kms_manager = Arc::new(KmsManager::new(backend, config.clone()));
// Create encryption service
let encryption_service = Arc::new(ObjectEncryptionService::new((*kms_manager).clone()));
@@ -649,7 +596,6 @@ impl KmsServiceManager {
manager: kms_manager,
credential_task,
deletion_worker: None,
probe_worker: None,
})
}
@@ -663,13 +609,9 @@ impl KmsServiceManager {
fn publish_running(&self, config: KmsConfig, mut service_version: ServiceVersion) {
if let Some(previous) = self.state.load().current_service.as_ref() {
previous.shutdown_background_workers();
previous.shutdown_deletion_worker();
}
service_version.deletion_worker = self.spawn_deletion_worker(&config, &service_version);
// Started at publish time for the same reason as the deletion worker:
// a start or reconfigure candidate that never becomes current must not
// leak a running task or publish probe results nobody asked for.
service_version.probe_worker = spawn_probe_worker(service_version.manager.backend()).map(Arc::new);
self.state.store(Arc::new(RuntimeState {
config: Some(config),
status: KmsServiceStatus::Running,
@@ -687,13 +629,7 @@ impl KmsServiceManager {
return None;
}
let cancel = CancellationToken::new();
let worker = DeletionWorker::new(
backend,
config.default_key_id.clone(),
self.deletion_reference_checker(),
config.backend.as_str(),
)
.with_audit_sink(self.audit_sink());
let worker = DeletionWorker::new(backend, config.default_key_id.clone(), self.deletion_reference_checker());
let task = worker.spawn(cancel.clone());
Some(Arc::new(DeletionWorkerHandle {
cancel,
@@ -1044,49 +980,6 @@ mod tests {
);
}
#[tokio::test]
async fn probe_worker_follows_the_service_lifecycle() {
use tempfile::TempDir;
let key_dir = TempDir::new().expect("create local KMS directory");
let mut config = KmsConfig::local(key_dir.path().to_path_buf());
config.allow_insecure_dev_defaults = true;
let manager = KmsServiceManager::new();
manager.configure(config).await.expect("configure local KMS");
manager.start().await.expect("start local KMS");
let first_probe = manager
.state
.load()
.current_service
.as_ref()
.expect("running service")
.probe_worker
.clone()
.expect("a backend with a data key round trip must run a probe");
assert!(!first_probe.is_cancelled());
assert!(manager.probe_status().is_some(), "a running service must publish probe status");
// Replacing the service version replaces (and cancels) its probe.
manager.restart().await.expect("restart");
assert!(first_probe.is_cancelled(), "replaced version's probe must be cancelled");
let second_probe = manager
.state
.load()
.current_service
.as_ref()
.expect("running service")
.probe_worker
.clone()
.expect("restarted service must run a fresh probe");
assert!(!second_probe.is_cancelled());
// Stopping the service stops its probe and withdraws the snapshot.
manager.stop().await.expect("stop");
assert!(second_probe.is_cancelled(), "stop must cancel the probe worker");
assert!(manager.probe_status().is_none(), "a stopped service must publish no status");
}
#[tokio::test]
async fn reconfigure_allows_safe_local_runtime_settings_only() {
use tempfile::TempDir;
-14
View File
@@ -484,20 +484,6 @@ impl OperationContext {
}
}
/// Principal recorded for work the server starts on its own behalf.
///
/// Namespaced so it can never collide with an access key or an IAM ARN,
/// which keeps "no human did this" distinguishable from "we lost track of
/// who did this" in the audit trail.
pub const INTERNAL_PRINCIPAL: &'static str = "rustfs:internal";
/// Context for an operation with no authenticated caller, such as
/// background maintenance or a call made while serving a data-plane
/// request that carries its own audit entry.
pub fn internal() -> Self {
Self::new(Self::INTERNAL_PRINCIPAL.to_string())
}
/// Add additional context
///
/// # Arguments
-2
View File
@@ -22,8 +22,6 @@ mod pattern_rules_test;
#[cfg(test)]
mod pattern_test;
mod rules_map;
#[cfg(test)]
mod rules_map_test;
mod subscriber_index;
mod subscriber_snapshot;
mod target_id_set;
-104
View File
@@ -1,104 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Regressions for rule matching against the KMS event namespace.
//!
//! KMS events (backlog#1583) are appended to `EventName` for the audit sink,
//! but they must stay invisible to bucket notification rules: a subscriber
//! asking for `s3:ObjectCreated:*` — or for everything — must never receive
//! key management activity.
use super::RulesMap;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::TargetID;
const KMS_EVENTS: &[EventName] = &[
EventName::KmsKeyCreated,
EventName::KmsKeyRotated,
EventName::KmsKeyEnabled,
EventName::KmsKeyDisabled,
EventName::KmsKeyDeletionScheduled,
EventName::KmsKeyDeletionCancelled,
EventName::KmsKeyDeleted,
EventName::KmsKeyAccessed,
];
fn test_target() -> TargetID {
TargetID::new("primary".to_string(), "webhook".to_string())
}
fn rules_for(events: &[EventName]) -> RulesMap {
let mut rules = RulesMap::new();
rules.add_rule_config(events, String::new(), test_target());
rules
}
#[test]
fn s3_wildcard_subscriptions_do_not_match_kms_events() {
let rules = rules_for(&[
EventName::ObjectCreatedAll,
EventName::ObjectAccessedAll,
EventName::ObjectRemovedAll,
EventName::ObjectTaggingAll,
EventName::ObjectReplicationAll,
EventName::ObjectRestoreAll,
EventName::ObjectTransitionAll,
EventName::LifecycleExpirationAll,
EventName::ObjectScannerAll,
]);
for event in KMS_EVENTS {
assert!(!rules.has_subscriber(event), "{event} must not have an S3 wildcard subscriber");
assert!(
rules.match_rules(*event, "any/object").is_empty(),
"{event} must not match any S3 wildcard rule"
);
}
}
#[test]
fn everything_subscription_does_not_match_kms_events() {
let rules = rules_for(&[EventName::Everything]);
// Sanity: the catch-all really is subscribed to the S3 surface.
assert!(rules.has_subscriber(&EventName::ObjectCreatedPut));
for event in KMS_EVENTS {
assert!(!rules.has_subscriber(event), "{event} must stay outside the s3 catch-all");
assert!(
rules.match_rules(*event, "any/object").is_empty(),
"{event} must not match the s3 catch-all rule"
);
}
}
#[test]
fn kms_subscription_does_not_leak_into_s3_matching() {
// The inverse direction: even an explicit KMS subscription must not widen
// the mask so that S3 events start matching a KMS-only rule.
let rules = rules_for(KMS_EVENTS);
for event in [
EventName::ObjectCreatedPut,
EventName::ObjectRemovedDelete,
EventName::ObjectAccessedGet,
EventName::BucketCreated,
] {
assert!(!rules.has_subscriber(&event), "{event} must not match a KMS-only rule");
assert!(
rules.match_rules(event, "any/object").is_empty(),
"{event} must not match a KMS-only rule"
);
}
}
-2
View File
@@ -146,7 +146,6 @@ tracing-opentelemetry = { workspace = true }
tracing-subscriber = { workspace = true, features = ["fmt", "env-filter", "tracing-log", "local-time", "json"] }
tokio = { workspace = true, features = ["sync", "rt-multi-thread", "time", "macros"] }
tokio-util = { workspace = true, features = ["io", "compat"] }
url.workspace = true
dial9-tokio-telemetry = { workspace = true, optional = true }
thiserror = { workspace = true }
zstd = { workspace = true, features = ["zstdmt"] }
@@ -163,4 +162,3 @@ libc = { workspace = true }
[dev-dependencies]
tempfile = { workspace = true }
temp-env = { workspace = true }
log = "0.4"
@@ -16,25 +16,22 @@
use crate::metrics::report::PrometheusMetric;
use crate::metrics::schema::bucket_replication::{
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD,
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, BUCKET_REPL_LAST_HR_FAILED_COUNT_MD,
BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD, BUCKET_REPL_LATENCY_MS_MD,
BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD,
BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD,
BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD,
BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD,
BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD,
BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD,
BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD,
BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD,
BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L,
TARGET_ARN_L,
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD,
BUCKET_REPL_LAST_HR_FAILED_COUNT_MD, BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD,
BUCKET_REPL_LATENCY_MS_MD, BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD,
BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD,
BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, BUCKET_REPL_RESYNC_FAILED_TOTAL_MD,
BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD,
BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, TARGET_ARN_L,
};
use std::borrow::Cow;
const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 25;
const BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET: usize = 5;
#[derive(Debug, Clone, Default)]
pub struct BucketReplicationTargetStats {
@@ -83,16 +80,6 @@ pub struct BucketReplicationStats {
pub targets: Vec<BucketReplicationTargetStats>,
}
#[derive(Debug, Clone, Default)]
pub(crate) struct BucketReplicationBacklogStats {
pub(crate) bucket: String,
pub(crate) current_backlog_count: u64,
pub(crate) current_backlog_bytes: u64,
pub(crate) durable_mrf_available: bool,
pub(crate) durable_mrf_backlog_count: u64,
pub(crate) durable_mrf_backlog_bytes: u64,
}
pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
@@ -262,6 +249,7 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, stat.resync_duration_ms as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
for target in &stat.targets {
let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
metrics.push(
@@ -277,43 +265,6 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
metrics
}
pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicationBacklogStats]) -> Vec<PrometheusMetric> {
if stats.is_empty() {
return Vec::new();
}
let mut metrics = Vec::with_capacity(stats.len() * BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET);
for stat in stats {
let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, stat.current_backlog_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD, stat.current_backlog_bytes as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(
&BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD,
if stat.durable_mrf_available { 1.0 } else { 0.0 },
)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, stat.durable_mrf_backlog_count as f64)
.with_label(BUCKET_L, bucket_label.clone()),
);
metrics.push(
PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD, stat.durable_mrf_backlog_bytes as f64)
.with_label(BUCKET_L, bucket_label),
);
}
metrics
}
#[cfg(test)]
mod tests {
use super::*;
@@ -418,56 +369,6 @@ mod tests {
}));
}
#[test]
fn test_collect_bucket_replication_backlog_metrics() {
let stats = vec![BucketReplicationBacklogStats {
bucket: "b1".to_string(),
current_backlog_count: 3,
current_backlog_bytes: 4096,
durable_mrf_available: true,
durable_mrf_backlog_count: 2,
durable_mrf_backlog_bytes: 2048,
}];
let metrics = collect_bucket_replication_backlog_metrics(&stats);
assert_eq!(metrics.len(), 5);
let backlog_count_name = BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == backlog_count_name
&& metric.value == 3.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
let backlog_bytes_name = BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == backlog_bytes_name
&& metric.value == 4096.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
let durable_available_name = BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == durable_available_name
&& metric.value == 1.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
let durable_count_name = BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == durable_count_name
&& metric.value == 2.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
let durable_bytes_name = BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD.get_full_metric_name();
assert!(metrics.iter().any(|metric| {
metric.name == durable_bytes_name
&& metric.value == 2048.0
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
}));
}
#[test]
fn test_collect_bucket_replication_metrics_empty() {
let stats: Vec<BucketReplicationStats> = Vec::new();
@@ -475,41 +376,6 @@ mod tests {
assert!(metrics.is_empty());
}
#[test]
fn backlog_metrics_are_bucket_scoped_without_target_labels() {
let stats = vec![BucketReplicationBacklogStats {
bucket: "scope-bucket".to_string(),
current_backlog_count: 5,
current_backlog_bytes: 8192,
durable_mrf_available: true,
durable_mrf_backlog_count: 2,
durable_mrf_backlog_bytes: 4096,
}];
let metrics = collect_bucket_replication_backlog_metrics(&stats);
let backlog_names = [
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD.get_full_metric_name(),
BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD.get_full_metric_name(),
BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD.get_full_metric_name(),
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD.get_full_metric_name(),
BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD.get_full_metric_name(),
];
for name in backlog_names {
let metric = metrics
.iter()
.find(|metric| metric.name == name)
.expect("backlog metric should be emitted");
assert!(
metric
.labels
.iter()
.any(|(key, value)| *key == BUCKET_L && value == "scope-bucket")
);
assert!(!metric.labels.iter().any(|(key, _)| *key == TARGET_ARN_L));
}
}
#[test]
fn test_collect_bucket_replication_bandwidth_metrics() {
let stats = vec![BucketReplicationBandwidthStats {
-1
View File
@@ -42,7 +42,6 @@ pub mod system_process;
pub use audit::{AuditTargetStats, collect_audit_metrics};
pub use bucket::{BucketStats, collect_bucket_metrics};
pub(crate) use bucket_replication::{BucketReplicationBacklogStats, collect_bucket_replication_backlog_metrics};
pub use bucket_replication::{
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
+3 -3
View File
@@ -31,9 +31,9 @@ pub use scheduler::{
metrics_runtime_controller_snapshot, metrics_runtime_status_snapshot,
};
pub(crate) use storage_api::metrics::{
BucketOperations, BucketOptions, ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore,
StorageAdminApi, obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor,
obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
BucketOperations, BucketOptions, ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsStore, StorageAdminApi,
obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config,
obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
obs_resolve_object_store_handle, obs_transition_state_handle,
};
+2 -4
View File
@@ -35,7 +35,6 @@ use crate::metrics::collectors::{
ProcessMemoryStats,
collect_audit_metrics,
collect_bucket_metrics,
collect_bucket_replication_backlog_metrics,
collect_bucket_replication_bandwidth_metrics,
collect_bucket_replication_metrics,
collect_bucket_usage_metrics,
@@ -95,7 +94,7 @@ use crate::metrics::schema::notification_target::{
};
use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL};
use crate::metrics::stats_collector::{
ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_stats_bundle,
ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_detail_stats,
collect_bucket_stats, collect_cluster_and_health_stats, collect_cluster_config_stats, collect_cluster_usage_metric_stats,
collect_compression_cluster_stats, collect_disk_and_system_drive_stats, collect_erasure_set_stats,
collect_host_network_stats, collect_iam_stats, collect_ilm_metric_stats, collect_internode_network_stats,
@@ -1349,9 +1348,8 @@ pub fn init_metrics_runtime(token: CancellationToken) {
// Phase-1 action: force zero for removed keys during tombstone cycles.
metrics.extend(collect_repl_bw_zero_tombstone_metrics(&zero_tombstones));
let (bucket_replication, bucket_replication_backlog) = collect_bucket_replication_stats_bundle().await;
let bucket_replication = collect_bucket_replication_detail_stats().await;
metrics.extend(collect_bucket_replication_metrics(&bucket_replication));
metrics.extend(collect_bucket_replication_backlog_metrics(&bucket_replication_backlog));
let replication = collect_replication_stats().await;
metrics.extend(collect_replication_metrics(&replication));
report_metrics(&metrics);
@@ -33,11 +33,6 @@ const RESYNC_COMPLETED_TOTAL: &str = "resync_completed_total";
const RESYNC_FAILED_TOTAL: &str = "resync_failed_total";
const RESYNC_CANCELED_TOTAL: &str = "resync_canceled_total";
const RESYNC_DURATION_MS_TOTAL: &str = "resync_duration_ms_total";
const CURRENT_BACKLOG_COUNT: &str = "current_backlog_count";
const CURRENT_BACKLOG_BYTES: &str = "current_backlog_bytes";
const DURABLE_MRF_AVAILABLE: &str = "durable_mrf_available";
const DURABLE_MRF_BACKLOG_COUNT: &str = "durable_mrf_backlog_count";
const DURABLE_MRF_BACKLOG_BYTES: &str = "durable_mrf_backlog_bytes";
pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
@@ -84,51 +79,6 @@ pub static BUCKET_REPL_LATENCY_MS_MD: LazyLock<MetricDescriptor> = LazyLock::new
)
});
pub static BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(CURRENT_BACKLOG_BYTES),
"Current number of bytes admitted to the in-memory replication worker queues for a bucket",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(CURRENT_BACKLOG_COUNT),
"Current number of objects admitted to the in-memory replication worker queues for a bucket",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(DURABLE_MRF_AVAILABLE),
"Whether the durable MRF backlog snapshot is available for this bucket on this node",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(DURABLE_MRF_BACKLOG_COUNT),
"Current number of objects in the durable MRF backlog file for a bucket on this node",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::from(DURABLE_MRF_BACKLOG_BYTES),
"Current bytes in the durable MRF backlog file for a bucket on this node",
&[BUCKET_L],
subsystems::BUCKET_REPLICATION,
)
});
pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_counter_md(
MetricName::ProxiedDeleteTaggingRequestsTotal,
+1 -1
View File
@@ -128,7 +128,7 @@ pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = L
pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
new_gauge_md(
MetricName::ReplicationRecentBacklogCount,
"Legacy replication backlog indicator: failed target objects plus objects currently queued on this node",
"Total number of objects currently in replication backlog (failed plus queued)",
&[],
subsystems::REPLICATION,
)
+52 -78
View File
@@ -21,18 +21,17 @@
//! and convert them to the Stats structs used by collectors.
use crate::metrics::collectors::{
BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats,
CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats, HostNetworkStats,
IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats,
ScannerStats,
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats, BucketStats, BucketUsageStats,
ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats,
DriveCountStats, DriveDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats,
ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats, ScannerStats,
};
use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
use crate::metrics::{
BucketOperations, BucketOptions, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore, StorageAdminApi,
obs_bucket_replication_stats_snapshot, obs_get_quota_config, obs_get_total_usable_capacity,
obs_get_total_usable_capacity_free, obs_load_compression_total_from_memory, obs_load_data_usage_from_backend,
obs_replication_site_stats_snapshot, obs_resolve_object_store_handle,
BucketOperations, BucketOptions, ObsEcstoreResult, ObsStore, StorageAdminApi, obs_bucket_replication_stats_snapshot,
obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free,
obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
obs_resolve_object_store_handle,
};
use crate::node_identity::current_local_node_identity;
use chrono::Utc;
@@ -193,65 +192,49 @@ async fn obs_ilm_runtime_snapshot() -> ObsIlmRuntimeSnapshot {
ilm_runtime_snapshot().await
}
async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>) {
let snapshots = obs_bucket_replication_stats_snapshot().await;
let mut detail_stats = Vec::with_capacity(snapshots.len());
let mut backlog_stats = Vec::with_capacity(snapshots.len());
for stats in snapshots {
backlog_stats.push(BucketReplicationBacklogStats {
bucket: stats.bucket.clone(),
current_backlog_count: stats.current_backlog_count,
current_backlog_bytes: stats.current_backlog_bytes,
durable_mrf_available: stats.durable_mrf_available,
durable_mrf_backlog_count: stats.durable_mrf_backlog_count,
durable_mrf_backlog_bytes: stats.durable_mrf_backlog_bytes,
});
detail_stats.push(bucket_replication_detail_from_snapshot(stats));
}
(detail_stats, backlog_stats)
}
fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnapshot) -> BucketReplicationStats {
BucketReplicationStats {
bucket: stats.bucket,
total_failed_bytes: stats.total_failed_bytes,
total_failed_count: stats.total_failed_count,
last_min_failed_bytes: stats.last_min_failed_bytes,
last_min_failed_count: stats.last_min_failed_count,
last_hour_failed_bytes: stats.last_hour_failed_bytes,
last_hour_failed_count: stats.last_hour_failed_count,
sent_bytes: stats.sent_bytes,
sent_count: stats.sent_count,
proxied_get_requests_total: stats.proxied_get_requests_total,
proxied_get_requests_failures: stats.proxied_get_requests_failures,
proxied_head_requests_total: stats.proxied_head_requests_total,
proxied_head_requests_failures: stats.proxied_head_requests_failures,
proxied_put_requests_total: stats.proxied_put_requests_total,
proxied_put_requests_failures: stats.proxied_put_requests_failures,
proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
resync_started_count: stats.resync_started_count,
resync_completed_count: stats.resync_completed_count,
resync_failed_count: stats.resync_failed_count,
resync_canceled_count: stats.resync_canceled_count,
resync_duration_ms: stats.resync_duration_ms,
targets: stats
.targets
.into_iter()
.map(|target| BucketReplicationTargetStats {
target_arn: target.target_arn,
bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
latency_ms: target.latency_ms,
})
.collect(),
}
async fn obs_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
obs_bucket_replication_stats_snapshot()
.await
.into_iter()
.map(|stats| BucketReplicationStats {
bucket: stats.bucket,
total_failed_bytes: stats.total_failed_bytes,
total_failed_count: stats.total_failed_count,
last_min_failed_bytes: stats.last_min_failed_bytes,
last_min_failed_count: stats.last_min_failed_count,
last_hour_failed_bytes: stats.last_hour_failed_bytes,
last_hour_failed_count: stats.last_hour_failed_count,
sent_bytes: stats.sent_bytes,
sent_count: stats.sent_count,
proxied_get_requests_total: stats.proxied_get_requests_total,
proxied_get_requests_failures: stats.proxied_get_requests_failures,
proxied_head_requests_total: stats.proxied_head_requests_total,
proxied_head_requests_failures: stats.proxied_head_requests_failures,
proxied_put_requests_total: stats.proxied_put_requests_total,
proxied_put_requests_failures: stats.proxied_put_requests_failures,
proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
resync_started_count: stats.resync_started_count,
resync_completed_count: stats.resync_completed_count,
resync_failed_count: stats.resync_failed_count,
resync_canceled_count: stats.resync_canceled_count,
resync_duration_ms: stats.resync_duration_ms,
targets: stats
.targets
.into_iter()
.map(|target| BucketReplicationTargetStats {
target_arn: target.target_arn,
bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
latency_ms: target.latency_ms,
})
.collect(),
})
.collect()
}
async fn obs_site_replication_stats() -> ReplicationStats {
@@ -543,16 +526,7 @@ pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBand
/// Collect bucket and target level replication stats from the global replication runtime.
pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
obs_bucket_replication_stats_snapshot()
.await
.into_iter()
.map(bucket_replication_detail_from_snapshot)
.collect()
}
pub(crate) async fn collect_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>)
{
obs_bucket_replication_stats_bundle().await
obs_bucket_replication_detail_stats().await
}
/// Collect site-level replication stats from the global replication runtime.
+62 -268
View File
@@ -12,14 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use std::time::Duration;
pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
use rustfs_ecstore::api::bucket::replication::{
DurableMrfBucketBacklog, durable_mrf_backlog_summary_snapshot, get_global_replication_stats,
};
use rustfs_ecstore::api::bucket::replication::get_global_replication_stats;
pub(crate) use rustfs_ecstore::api::capacity::{
get_total_usable_capacity as obs_get_total_usable_capacity,
get_total_usable_capacity_free as obs_get_total_usable_capacity_free,
@@ -71,50 +68,9 @@ pub(crate) struct ObsBucketReplicationStatsSnapshot {
pub(crate) resync_failed_count: u64,
pub(crate) resync_canceled_count: u64,
pub(crate) resync_duration_ms: u64,
pub(crate) current_backlog_count: u64,
pub(crate) current_backlog_bytes: u64,
pub(crate) durable_mrf_available: bool,
pub(crate) durable_mrf_backlog_count: u64,
pub(crate) durable_mrf_backlog_bytes: u64,
pub(crate) targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
}
#[derive(Debug, Clone, Default, PartialEq)]
struct ObsBucketReplicationRuntimeSnapshot {
total_failed_bytes: u64,
total_failed_count: u64,
last_min_failed_bytes: u64,
last_min_failed_count: u64,
last_hour_failed_bytes: u64,
last_hour_failed_count: u64,
sent_bytes: u64,
sent_count: u64,
resync_started_count: u64,
resync_completed_count: u64,
resync_failed_count: u64,
resync_canceled_count: u64,
resync_duration_ms: u64,
current_backlog_count: u64,
current_backlog_bytes: u64,
targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
}
#[derive(Debug, Clone, Default, PartialEq)]
struct ObsBucketReplicationProxySnapshot {
proxied_get_requests_total: u64,
proxied_get_requests_failures: u64,
proxied_head_requests_total: u64,
proxied_head_requests_failures: u64,
proxied_put_requests_total: u64,
proxied_put_requests_failures: u64,
proxied_put_tagging_requests_total: u64,
proxied_put_tagging_requests_failures: u64,
proxied_get_tagging_requests_total: u64,
proxied_get_tagging_requests_failures: u64,
proxied_delete_tagging_requests_total: u64,
proxied_delete_tagging_requests_failures: u64,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub(crate) struct ObsReplicationSiteStatsSnapshot {
pub(crate) average_active_workers: f64,
@@ -146,85 +102,59 @@ fn replication_backlog_count(failed_counts: impl Iterator<Item = i64>, queued_co
failed_backlog.saturating_add(i64_to_u64_floor_zero(queued_count))
}
fn bucket_replication_stats_snapshot_from_parts(
bucket: String,
runtime: ObsBucketReplicationRuntimeSnapshot,
proxy: ObsBucketReplicationProxySnapshot,
durable_mrf_available: bool,
durable_bucket: DurableMrfBucketBacklog,
) -> ObsBucketReplicationStatsSnapshot {
ObsBucketReplicationStatsSnapshot {
bucket,
total_failed_bytes: runtime.total_failed_bytes,
total_failed_count: runtime.total_failed_count,
last_min_failed_bytes: runtime.last_min_failed_bytes,
last_min_failed_count: runtime.last_min_failed_count,
last_hour_failed_bytes: runtime.last_hour_failed_bytes,
last_hour_failed_count: runtime.last_hour_failed_count,
sent_bytes: runtime.sent_bytes,
sent_count: runtime.sent_count,
proxied_get_requests_total: proxy.proxied_get_requests_total,
proxied_get_requests_failures: proxy.proxied_get_requests_failures,
proxied_head_requests_total: proxy.proxied_head_requests_total,
proxied_head_requests_failures: proxy.proxied_head_requests_failures,
proxied_put_requests_total: proxy.proxied_put_requests_total,
proxied_put_requests_failures: proxy.proxied_put_requests_failures,
proxied_put_tagging_requests_total: proxy.proxied_put_tagging_requests_total,
proxied_put_tagging_requests_failures: proxy.proxied_put_tagging_requests_failures,
proxied_get_tagging_requests_total: proxy.proxied_get_tagging_requests_total,
proxied_get_tagging_requests_failures: proxy.proxied_get_tagging_requests_failures,
proxied_delete_tagging_requests_total: proxy.proxied_delete_tagging_requests_total,
proxied_delete_tagging_requests_failures: proxy.proxied_delete_tagging_requests_failures,
resync_started_count: runtime.resync_started_count,
resync_completed_count: runtime.resync_completed_count,
resync_failed_count: runtime.resync_failed_count,
resync_canceled_count: runtime.resync_canceled_count,
resync_duration_ms: runtime.resync_duration_ms,
current_backlog_count: runtime.current_backlog_count,
current_backlog_bytes: runtime.current_backlog_bytes,
durable_mrf_available,
durable_mrf_backlog_count: durable_bucket.count,
durable_mrf_backlog_bytes: durable_bucket.bytes,
targets: runtime.targets,
}
}
pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketReplicationStatsSnapshot> {
let stats = get_global_replication_stats();
let all_bucket_stats = if let Some(stats) = &stats {
stats.get_all().await
} else {
HashMap::new()
let Some(stats) = get_global_replication_stats() else {
return Vec::new();
};
let durable_mrf_summary = if obs_resolve_object_store_handle().is_some() {
durable_mrf_backlog_summary_snapshot()
} else {
Default::default()
};
let durable_mrf_available = durable_mrf_summary.available;
let durable_buckets = durable_mrf_summary
.buckets
.into_iter()
.map(|bucket| (bucket.bucket.clone(), bucket))
.collect::<HashMap<String, DurableMrfBucketBacklog>>();
let mut bucket_names = Vec::with_capacity(all_bucket_stats.len().saturating_add(durable_buckets.len()));
bucket_names.extend(all_bucket_stats.keys().cloned());
bucket_names.extend(
durable_buckets
.keys()
.filter(|bucket| !all_bucket_stats.contains_key(*bucket))
.cloned(),
);
let mut buckets = Vec::with_capacity(bucket_names.len());
for bucket in bucket_names {
let bucket_stats = all_bucket_stats.get(&bucket);
let proxy = if let Some(stats) = &stats {
stats.get_proxy_stats(&bucket).await
} else {
Default::default()
};
let proxy = ObsBucketReplicationProxySnapshot {
let all_bucket_stats = stats.get_all().await;
let mut buckets = Vec::with_capacity(all_bucket_stats.len());
for (bucket, bucket_stats) in all_bucket_stats {
let proxy = stats.get_proxy_stats(&bucket).await;
let mut total_failed_bytes = 0u64;
let mut total_failed_count = 0u64;
let mut last_min_failed_bytes = 0u64;
let mut last_min_failed_count = 0u64;
let mut last_hour_failed_bytes = 0u64;
let mut last_hour_failed_count = 0u64;
let mut sent_bytes = 0u64;
let mut sent_count = 0u64;
let mut targets = Vec::with_capacity(bucket_stats.stats.len());
for (target_arn, target_stats) in bucket_stats.stats {
total_failed_bytes = total_failed_bytes.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
total_failed_count = total_failed_count.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
last_min_failed_bytes = last_min_failed_bytes.saturating_add(i64_to_u64_floor_zero(last_min.size));
last_min_failed_count = last_min_failed_count.saturating_add(i64_to_u64_floor_zero(last_min.count));
let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
last_hour_failed_bytes = last_hour_failed_bytes.saturating_add(i64_to_u64_floor_zero(last_hour.size));
last_hour_failed_count = last_hour_failed_count.saturating_add(i64_to_u64_floor_zero(last_hour.count));
sent_bytes = sent_bytes.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
sent_count = sent_count.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
targets.push(ObsBucketReplicationTargetStatsSnapshot {
target_arn,
bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
latency_ms: target_stats.latency.curr,
});
}
buckets.push(ObsBucketReplicationStatsSnapshot {
bucket,
total_failed_bytes,
total_failed_count,
last_min_failed_bytes,
last_min_failed_count,
last_hour_failed_bytes,
last_hour_failed_count,
sent_bytes,
sent_count,
proxied_get_requests_total: i64_to_u64_floor_zero(proxy.get_total),
proxied_get_requests_failures: i64_to_u64_floor_zero(proxy.get_failed),
proxied_head_requests_total: i64_to_u64_floor_zero(proxy.head_total),
@@ -237,67 +167,13 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
proxied_get_tagging_requests_failures: i64_to_u64_floor_zero(proxy.get_tag_failed),
proxied_delete_tagging_requests_total: i64_to_u64_floor_zero(proxy.delete_tag_total),
proxied_delete_tagging_requests_failures: i64_to_u64_floor_zero(proxy.delete_tag_failed),
};
let mut runtime = ObsBucketReplicationRuntimeSnapshot {
targets: Vec::with_capacity(bucket_stats.map(|stats| stats.stats.len()).unwrap_or(0)),
..Default::default()
};
if let Some(bucket_stats) = bucket_stats {
for (target_arn, target_stats) in &bucket_stats.stats {
runtime.total_failed_bytes = runtime
.total_failed_bytes
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
runtime.total_failed_count = runtime
.total_failed_count
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
runtime.last_min_failed_bytes = runtime
.last_min_failed_bytes
.saturating_add(i64_to_u64_floor_zero(last_min.size));
runtime.last_min_failed_count = runtime
.last_min_failed_count
.saturating_add(i64_to_u64_floor_zero(last_min.count));
let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
runtime.last_hour_failed_bytes = runtime
.last_hour_failed_bytes
.saturating_add(i64_to_u64_floor_zero(last_hour.size));
runtime.last_hour_failed_count = runtime
.last_hour_failed_count
.saturating_add(i64_to_u64_floor_zero(last_hour.count));
runtime.sent_bytes = runtime
.sent_bytes
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
runtime.sent_count = runtime
.sent_count
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
runtime.targets.push(ObsBucketReplicationTargetStatsSnapshot {
target_arn: target_arn.clone(),
bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
latency_ms: target_stats.latency.curr,
});
}
runtime.resync_started_count = i64_to_u64_floor_zero(bucket_stats.resync_started_count);
runtime.resync_completed_count = i64_to_u64_floor_zero(bucket_stats.resync_completed_count);
runtime.resync_failed_count = i64_to_u64_floor_zero(bucket_stats.resync_failed_count);
runtime.resync_canceled_count = i64_to_u64_floor_zero(bucket_stats.resync_canceled_count);
runtime.resync_duration_ms = i64_to_u64_floor_zero(bucket_stats.resync_duration_ms);
runtime.current_backlog_count = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.count);
runtime.current_backlog_bytes = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.bytes);
}
let durable_bucket = durable_buckets.get(&bucket).cloned().unwrap_or_default();
buckets.push(bucket_replication_stats_snapshot_from_parts(
bucket,
runtime,
proxy,
durable_mrf_available,
durable_bucket,
));
resync_started_count: i64_to_u64_floor_zero(bucket_stats.resync_started_count),
resync_completed_count: i64_to_u64_floor_zero(bucket_stats.resync_completed_count),
resync_failed_count: i64_to_u64_floor_zero(bucket_stats.resync_failed_count),
resync_canceled_count: i64_to_u64_floor_zero(bucket_stats.resync_canceled_count),
resync_duration_ms: i64_to_u64_floor_zero(bucket_stats.resync_duration_ms),
targets,
});
}
buckets
@@ -367,97 +243,15 @@ mod tests {
fn replication_backlog_count_floors_negative_failed_and_queue_values() {
assert_eq!(replication_backlog_count([-3, 4].into_iter(), -2), 4);
}
#[test]
fn replication_backlog_count_keeps_legacy_failed_backlog_semantics() {
assert_eq!(replication_backlog_count([9].into_iter(), 0), 9);
}
#[test]
fn bucket_replication_snapshot_maps_runtime_and_durable_backlog() {
let snapshot = bucket_replication_stats_snapshot_from_parts(
"runtime-bucket".to_string(),
ObsBucketReplicationRuntimeSnapshot {
current_backlog_count: 3,
current_backlog_bytes: 4096,
resync_failed_count: 2,
..Default::default()
},
ObsBucketReplicationProxySnapshot {
proxied_get_requests_total: 7,
proxied_get_requests_failures: 1,
..Default::default()
},
true,
DurableMrfBucketBacklog {
bucket: "runtime-bucket".to_string(),
count: 5,
bytes: 8192,
},
);
assert_eq!(snapshot.bucket, "runtime-bucket");
assert_eq!(snapshot.current_backlog_count, 3);
assert_eq!(snapshot.current_backlog_bytes, 4096);
assert!(snapshot.durable_mrf_available);
assert_eq!(snapshot.durable_mrf_backlog_count, 5);
assert_eq!(snapshot.durable_mrf_backlog_bytes, 8192);
assert_eq!(snapshot.resync_failed_count, 2);
assert_eq!(snapshot.proxied_get_requests_total, 7);
assert_eq!(snapshot.proxied_get_requests_failures, 1);
}
#[test]
fn bucket_replication_snapshot_reports_durable_only_bucket() {
let snapshot = bucket_replication_stats_snapshot_from_parts(
"durable-only".to_string(),
ObsBucketReplicationRuntimeSnapshot::default(),
ObsBucketReplicationProxySnapshot::default(),
true,
DurableMrfBucketBacklog {
bucket: "durable-only".to_string(),
count: 11,
bytes: 2048,
},
);
assert_eq!(snapshot.bucket, "durable-only");
assert_eq!(snapshot.current_backlog_count, 0);
assert!(snapshot.durable_mrf_available);
assert_eq!(snapshot.durable_mrf_backlog_count, 11);
assert_eq!(snapshot.durable_mrf_backlog_bytes, 2048);
}
#[test]
fn bucket_replication_snapshot_preserves_durable_mrf_unavailable_state() {
let snapshot = bucket_replication_stats_snapshot_from_parts(
"runtime-only".to_string(),
ObsBucketReplicationRuntimeSnapshot {
current_backlog_count: 1,
current_backlog_bytes: 512,
..Default::default()
},
ObsBucketReplicationProxySnapshot::default(),
false,
DurableMrfBucketBacklog::default(),
);
assert_eq!(snapshot.current_backlog_count, 1);
assert_eq!(snapshot.current_backlog_bytes, 512);
assert!(!snapshot.durable_mrf_available);
assert_eq!(snapshot.durable_mrf_backlog_count, 0);
assert_eq!(snapshot.durable_mrf_backlog_bytes, 0);
}
}
pub(crate) mod metrics {
pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, StorageAdminApi};
pub(crate) use super::{
ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore,
obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config,
obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
obs_resolve_object_store_handle, obs_transition_state_handle,
ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsStore, obs_bucket_replication_stats_snapshot, obs_expiry_state_handle,
obs_get_global_bucket_monitor, obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free,
obs_is_disk_compression_enabled, obs_load_compression_total_from_memory, obs_load_data_usage_from_backend,
obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, obs_transition_state_handle,
};
}
+1 -20
View File
@@ -18,26 +18,7 @@
//! used across different logging backends (stdout, file, OpenTelemetry).
use std::env;
use tracing::Metadata;
use tracing_subscriber::{
EnvFilter,
filter::{FilterFn, LevelFilter, filter_fn},
};
/// Pyroscope emits raw reqwest errors from its background session manager.
/// Those errors can include the configured endpoint, so they must never reach
/// a RustFS logging sink even when an operator enables verbose dependency logs.
pub(super) fn is_pyroscope_dependency_target(target: &str) -> bool {
target == "pyroscope" || target.starts_with("pyroscope::") || target.starts_with("Pyroscope::")
}
fn allows_non_pyroscope_dependency_event(metadata: &Metadata<'_>) -> bool {
!is_pyroscope_dependency_target(metadata.target())
}
pub(super) fn pyroscope_log_filter() -> FilterFn {
filter_fn(allows_non_pyroscope_dependency_event)
}
use tracing_subscriber::{EnvFilter, filter::LevelFilter};
/// Build an `EnvFilter` from the given log level string.
///
+1 -3
View File
@@ -36,7 +36,7 @@ use crate::cleaner::LogCleaner;
use crate::cleaner::types::{CompressionAlgorithm, FileMatchMode};
use crate::config::OtelConfig;
use crate::global::{METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL, METRIC_LOG_CLEANER_RUNS_TOTAL, set_observability_metric_enabled};
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter};
use crate::telemetry::filter::build_env_filter;
use crate::telemetry::rolling::{RollingAppender, Rotation};
use metrics::counter;
use rustfs_config::observability::{
@@ -319,7 +319,6 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
tracing_subscriber::registry()
.with(env_filter)
.with(pyroscope_log_filter())
.with(ErrorLayer::default())
.with(fmt_layer)
.try_init()
@@ -432,7 +431,6 @@ fn init_file_logging_internal(
tracing_subscriber::registry()
.with(env_filter)
.with(pyroscope_log_filter())
.with(ErrorLayer::default())
.with(file_layer)
.with(stdout_layer)
+21 -164
View File
@@ -39,7 +39,7 @@
use crate::cleaner::types::FileMatchMode;
use crate::config::OtelConfig;
use crate::global::set_observability_metric_enabled;
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter};
use crate::telemetry::filter::build_env_filter;
use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
use crate::telemetry::local::{build_json_log_layer, spawn_cleanup_task};
use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
@@ -68,7 +68,7 @@ use std::{fs, io::IsTerminal, time::Duration};
use tracing::{info, warn};
use tracing_error::ErrorLayer;
use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
use tracing_subscriber::{fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt};
use tracing_subscriber::{Layer, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt};
const GET_OBJECT_DURATION_HISTOGRAM_METRICS: &[&str] = &[
"rustfs_io_get_object_request_duration_seconds",
@@ -83,26 +83,6 @@ const GET_OBJECT_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
];
#[cfg(all(
feature = "pyroscope",
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
))]
const REDACTED_PROFILING_ENDPOINT: &str = "[redacted]";
#[cfg(all(
feature = "pyroscope",
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
))]
fn log_profiler_failure(result: &'static str, error_kind: &'static str) {
warn!(
backend = "pyroscope",
endpoint = REDACTED_PROFILING_ENDPOINT,
result,
error_kind,
"Profiling export agent initialization failed"
);
}
/// Initialize the full OpenTelemetry HTTP pipeline (traces + metrics + logs).
///
/// This function is invoked when at least one OTLP endpoint has been
@@ -178,7 +158,9 @@ pub(super) fn init_observability_http(
logger_provider = build_logger_provider(&log_ep, config, res, use_stdout)?;
// Build bridge to capture `tracing` events.
otel_bridge = logger_provider.as_ref().map(OpenTelemetryTracingBridge::new);
otel_bridge = logger_provider
.as_ref()
.map(|p| OpenTelemetryTracingBridge::new(p).with_filter(build_env_filter(logger_level, None)));
// No separate formatting layer is added here; when OTLP logging is
// active, the OpenTelemetry bridge is the authoritative sink for
@@ -274,7 +256,6 @@ pub(super) fn init_observability_http(
let filter = build_env_filter(logger_level, None);
tracing_subscriber::registry()
.with(filter)
.with(pyroscope_log_filter())
.with(ErrorLayer::default())
.with(file_layer_opt)
.with(stdout_layer_opt)
@@ -550,11 +531,6 @@ pub(super) fn init_profiler(config: &OtelConfig) -> Option<ProfilingAgent> {
return None;
};
if url::Url::parse(endpoint).is_err() {
log_profiler_failure("profiling_endpoint_invalid", "invalid_endpoint");
return None;
}
// Configure Pyroscope Agent
let backend = pprof_backend(PprofConfig::default(), BackendConfig::default());
let service_name = config.service_name.as_deref().unwrap_or(APP_NAME);
@@ -566,16 +542,28 @@ pub(super) fn init_profiler(config: &OtelConfig) -> Option<ProfilingAgent> {
.build()
{
Ok(agent) => agent,
Err(_) => {
log_profiler_failure("profiling_agent_build_failed", "build");
Err(err) => {
warn!(
backend = "pyroscope",
endpoint,
result = "profiling_agent_build_failed",
error = %err,
"Profiling export agent initialization failed"
);
return None;
}
};
match agent.start() {
Ok(agent) => Some(agent),
Err(_) => {
log_profiler_failure("profiling_agent_start_failed", "start");
Err(err) => {
warn!(
backend = "pyroscope",
endpoint,
result = "profiling_agent_start_failed",
error = ?err,
"Profiling export agent failed to start"
);
None
}
}
@@ -662,34 +650,6 @@ fn resolve_signal_timeout(common_timeout_millis: Option<u64>, signal_timeout_mil
#[cfg(test)]
mod tests {
use super::*;
use std::env;
use std::io::{self, Write};
use std::process::Command;
use std::sync::{Arc, Mutex};
const LOG_TRACER_CHILD_ENV: &str = "RUSTFS_OBS_LOG_TRACER_CHILD";
#[derive(Clone)]
struct TestWriter(Arc<Mutex<Vec<u8>>>);
impl Write for TestWriter {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.0.lock().expect("lock test log buffer").extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for TestWriter {
type Writer = Self;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
#[test]
/// Valid ratios should produce trace-id-ratio sampling.
@@ -806,107 +766,4 @@ mod tests {
assert_eq!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.first(), Some(&0.0001));
assert_eq!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.last(), Some(&10.0));
}
#[cfg(all(
feature = "pyroscope",
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
))]
#[test]
fn test_init_profiler_invalid_endpoint_redacts_sensitive_components() {
let endpoint = "https://profile-user:profile-token@10.24.0.5:invalid/private/profiles?access_token=query-secret";
let buffer = Arc::new(Mutex::new(Vec::new()));
let writer = TestWriter(Arc::clone(&buffer));
let subscriber = tracing_subscriber::fmt()
.with_ansi(false)
.without_time()
.with_writer(writer)
.finish();
let config = OtelConfig {
profiling_export_enabled: Some(true),
profiling_endpoint: Some(endpoint.to_string()),
..OtelConfig::default()
};
tracing::subscriber::with_default(subscriber, || assert!(init_profiler(&config).is_none()));
let rendered = String::from_utf8(buffer.lock().expect("lock test log buffer").clone()).expect("decode test log");
assert!(rendered.contains(REDACTED_PROFILING_ENDPOINT));
assert!(rendered.contains("error_kind=\"invalid_endpoint\""));
for leaked in [
"profile-user",
"profile-token",
"10.24.0.5",
"private/profiles",
"query-secret",
] {
assert!(!rendered.contains(leaked), "profiling log leaked {leaked}: {rendered}");
}
}
#[test]
fn test_pyroscope_upload_failure_targets_stay_filtered_when_env_filter_enables_them() {
let endpoint = "https://profile-user:profile-token@10.24.0.5/private/profiles?access_token=query-secret";
let buffer = Arc::new(Mutex::new(Vec::new()));
let writer = TestWriter(Arc::clone(&buffer));
let env_filter = tracing_subscriber::EnvFilter::new("pyroscope=trace")
.add_directive("trace".parse().expect("parse trace filter directive"))
.add_directive("Pyroscope::Session=trace".parse().expect("parse Pyroscope filter directive"));
let subscriber = tracing_subscriber::registry()
.with(env_filter)
.with(pyroscope_log_filter())
.with(
tracing_subscriber::fmt::layer()
.with_ansi(false)
.without_time()
.with_writer(writer),
);
tracing::subscriber::with_default(subscriber, || {
tracing::error!(target: "pyroscope::session", "SessionManager - Failed to send session: {endpoint}");
tracing::error!(target: "Pyroscope::Session", "SessionManager - Failed to send session: {endpoint}");
});
let rendered = String::from_utf8(buffer.lock().expect("lock test log buffer").clone()).expect("decode test log");
assert!(rendered.is_empty(), "filtered Pyroscope upload failure reached a log sink: {rendered}");
}
#[test]
fn test_pyroscope_log_facade_upload_failure_is_filtered() {
let endpoint = "https://profile-user:profile-token@10.24.0.5/private/profiles?access_token=query-secret";
if env::var_os(LOG_TRACER_CHILD_ENV).is_some() {
tracing_subscriber::registry()
.with(build_env_filter("info", None))
.with(pyroscope_log_filter())
.with(tracing_subscriber::fmt::layer().with_ansi(false).without_time())
.try_init()
.expect("install isolated LogTracer subscriber");
log::error!(target: "pyroscope::session", "SessionManager - Failed to send session: {endpoint}");
log::error!(target: "Pyroscope::Session", "SessionManager - Failed to send session: {endpoint}");
return;
}
let output = Command::new(env::current_exe().expect("resolve test executable"))
.args([
"--exact",
"telemetry::otel::tests::test_pyroscope_log_facade_upload_failure_is_filtered",
"--nocapture",
])
.env(LOG_TRACER_CHILD_ENV, "1")
.env("RUST_LOG", "trace,pyroscope=trace,Pyroscope::Session=trace")
.output()
.expect("run isolated LogTracer test process");
assert!(output.status.success(), "isolated LogTracer test failed: {output:?}");
let rendered = format!("{}{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr));
for leaked in [
"profile-user",
"profile-token",
"10.24.0.5",
"private/profiles",
"query-secret",
] {
assert!(!rendered.contains(leaked), "LogTracer path leaked {leaked}: {rendered}");
}
}
}
+1 -1
View File
@@ -32,6 +32,7 @@ workspace = true
default = []
hotpath = [
"hotpath/hotpath",
"hotpath/reqwest-0-13",
"rustfs-config/hotpath",
"rustfs-credentials/hotpath",
"rustfs-crypto/hotpath",
@@ -79,7 +80,6 @@ pollster.workspace = true
proptest = "1"
test-case.workspace = true
temp-env = { workspace = true }
tracing-subscriber = { workspace = true, features = ["fmt"] }
[lib]
doctest = false
+15 -225
View File
@@ -30,16 +30,17 @@ impl Args {
}
}
pub fn is_configured() -> bool {
env::var_os(ENV_POLICY_PLUGIN_OPA_URL).is_some_and(|url| !url.is_empty())
}
#[derive(Debug, Clone)]
pub struct AuthZPlugin {
client: reqwest::Client,
client: OpaHttpClient,
args: Args,
}
#[cfg(feature = "hotpath")]
type OpaHttpClient = hotpath::wrap::reqwest::Client;
#[cfg(not(feature = "hotpath"))]
type OpaHttpClient = reqwest::Client;
#[derive(Debug, thiserror::Error)]
pub enum OpaConfigError {
#[error("Missing required env var: {0}")]
@@ -58,44 +59,6 @@ pub enum OpaConfigError {
Connection(reqwest::Error),
}
impl OpaConfigError {
pub fn kind(&self) -> &'static str {
match self {
Self::MissingRequiredEnv(_) => "missing_required_env",
Self::InvalidEnvVars(_) => "invalid_env_vars",
Self::EnvRead { .. } => "env_read_failed",
Self::InvalidStatus(_) => "invalid_status",
Self::Connection(_) => "connection_failed",
}
}
}
fn redact_opa_connection_error(error: reqwest::Error) -> OpaConfigError {
OpaConfigError::Connection(error.without_url())
}
fn opa_request_error_kind(error: &reqwest::Error) -> &'static str {
if error.is_timeout() {
"timeout"
} else if error.is_connect() {
"connect"
} else if error.is_request() {
"request"
} else if error.is_body() {
"body"
} else if error.is_decode() {
"decode"
} else if error.is_status() {
"status"
} else if error.is_builder() {
"builder"
} else if error.is_redirect() {
"redirect"
} else {
"other"
}
}
fn check() -> Result<(), OpaConfigError> {
let env_list = env::vars();
let mut candidate = HashMap::new();
@@ -123,14 +86,9 @@ async fn validate(config: &Args) -> Result<(), OpaConfigError> {
.timeout(Duration::from_secs(5))
.connect_timeout(Duration::from_secs(1))
.build()
.map_err(redact_opa_connection_error)?;
.map_err(OpaConfigError::Connection)?;
let mut request = client.post(&config.url);
if !config.auth_token.is_empty() {
request = request.header("Authorization", format!("Bearer {}", config.auth_token));
}
match request.send().await {
match client.post(&config.url).send().await {
Ok(resp) => {
match resp.status() {
reqwest::StatusCode::OK => {
@@ -142,7 +100,7 @@ async fn validate(config: &Args) -> Result<(), OpaConfigError> {
};
}
Err(err) => {
return Err(redact_opa_connection_error(err));
return Err(OpaConfigError::Connection(err));
}
};
Ok(())
@@ -188,6 +146,9 @@ impl AuthZPlugin {
reqwest::Client::new()
});
#[cfg(feature = "hotpath")]
let client = hotpath::http!(client, label = "Policy::OPA");
Self { client, args: config }
}
@@ -203,13 +164,7 @@ impl AuthZPlugin {
Ok(resp) => {
let status = resp.status();
if !status.is_success() {
error!(
component = "policy",
subsystem = "opa",
result = "request_rejected",
status = %status,
"OPA request rejected"
);
error!("OPA returned non-success status: {}", status);
return false;
}
@@ -219,25 +174,13 @@ impl AuthZPlugin {
OpaResponseEnum::AllowResult(response) => response.result.allow,
},
Err(err) => {
error!(
component = "policy",
subsystem = "opa",
result = "response_decode_failed",
error_kind = opa_request_error_kind(&err),
"OPA response decoding failed"
);
error!("Error parsing OPA response: {:?}", err);
false
}
}
}
Err(err) => {
error!(
component = "policy",
subsystem = "opa",
result = "request_failed",
error_kind = opa_request_error_kind(&err),
"OPA request failed"
);
error!("Error sending request to OPA: {:?}", err);
false
}
}
@@ -306,35 +249,8 @@ enum OpaResponseEnum {
#[cfg(test)]
mod tests {
use super::*;
use std::io::{Read, Write};
use std::net::TcpListener;
use std::sync::{Arc, Mutex};
use temp_env;
#[derive(Clone)]
struct TestWriter(Arc<Mutex<Vec<u8>>>);
impl Write for TestWriter {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
self.0.lock().expect("lock test log buffer").extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for TestWriter {
type Writer = Self;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
fn assert_reqwest_client(_: &reqwest::Client) {}
#[test]
fn test_check_valid_config() {
// Use temp_env to temporarily set environment variables
@@ -420,130 +336,4 @@ mod tests {
};
assert!(!args_disabled.enable());
}
#[test]
fn test_sensitive_opa_endpoints_never_use_hotpath_http_wrapper() {
let endpoints = [
"https://profile-user:profile-token@10.24.0.5/private/opa?access_token=query-secret",
"https://service.internal.example/private/path?token=another-secret",
];
for endpoint in endpoints {
let plugin = AuthZPlugin::new(Args {
url: endpoint.to_string(),
auth_token: "opa-auth-token".to_string(),
});
assert_reqwest_client(&plugin.client);
assert_eq!(plugin.args.url, endpoint);
}
}
#[test]
fn test_opa_connection_error_removes_sensitive_endpoint() {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind local test listener");
let port = listener.local_addr().expect("read local test listener address").port();
let server = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().expect("accept local test request");
let mut request = [0_u8; 1024];
let read = stream.read(&mut request).expect("read local test request");
assert!(read > 0, "local test client must send an HTTP request");
stream
.write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.expect("write local test response");
});
let endpoint = format!("http://profile-user:profile-token@127.0.0.1:{port}/private/opa?access_token=query-secret");
let runtime = tokio::runtime::Runtime::new().expect("build tokio test runtime");
let error = runtime.block_on(async {
reqwest::Client::builder()
.no_proxy()
.build()
.expect("build local test client")
.get(&endpoint)
.send()
.await
.expect("send local test request")
.error_for_status()
.expect_err("test response should be an HTTP error")
});
server.join().expect("join local test server");
assert!(error.url().is_some(), "fixture must carry the endpoint before redaction");
let OpaConfigError::Connection(error) = redact_opa_connection_error(error) else {
panic!("expected a redacted OPA connection error");
};
let rendered = error.to_string();
assert!(error.url().is_none());
for leaked in ["profile-user", "profile-token", "127.0.0.1", "private/opa", "query-secret"] {
assert!(!rendered.contains(leaked), "redacted error leaked {leaked}: {rendered}");
}
}
#[test]
fn test_is_allowed_rejection_log_redacts_sensitive_endpoint() {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind local test listener");
let port = listener.local_addr().expect("read local test listener address").port();
let server = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().expect("accept local test request");
let mut request = [0_u8; 1024];
let read = stream.read(&mut request).expect("read local test request");
assert!(read > 0, "local test client must send an HTTP request");
stream
.write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.expect("write local test response");
});
let endpoint = format!("http://opa-user:opa-token@127.0.0.1:{port}/private/allow?access_token=query-secret");
let plugin = AuthZPlugin::new(Args {
url: endpoint,
auth_token: "authorization-secret".to_string(),
});
let groups = None;
let conditions = HashMap::new();
let claims = HashMap::new();
let args = PArgs {
account: "account",
groups: &groups,
action: crate::policy::action::Action::None,
bucket: "bucket",
conditions: &conditions,
is_owner: false,
object: "object",
claims: &claims,
deny_only: false,
};
let buffer = Arc::new(Mutex::new(Vec::new()));
let writer = TestWriter(Arc::clone(&buffer));
let subscriber = tracing_subscriber::fmt()
.with_ansi(false)
.without_time()
.with_writer(writer)
.finish();
tracing::subscriber::with_default(subscriber, || {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("build tokio test runtime");
assert!(!runtime.block_on(plugin.is_allowed(&args)));
});
server.join().expect("join local test server");
let rendered = String::from_utf8(buffer.lock().expect("lock test log buffer").clone()).expect("decode test log");
assert!(!rendered.is_empty(), "OPA request failure did not reach the test log sink");
assert!(
rendered.contains("request_rejected"),
"OPA request failure did not retain its stable result category"
);
assert!(rendered.contains("500 Internal Server Error"));
for leaked in [
"opa-user",
"opa-token",
"127.0.0.1",
"private/allow",
"query-secret",
"authorization-secret",
] {
assert!(!rendered.contains(leaked), "OPA request log leaked {leaked}: {rendered}");
}
}
}
@@ -1171,12 +1171,6 @@ pub struct SignalServiceResponse {
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
#[prost(uint32, tag = "3")]
pub protocol_version: u32,
/// Redacted fingerprint of the KMS configuration this node is running.
/// Only set for the "kms" dynamic config subsystem, and left unset when the
/// responder has no KMS configuration. Peers built before that subsystem
/// reject the signal outright rather than answering without this field.
#[prost(string, optional, tag = "4")]
pub config_fingerprint: ::core::option::Option<::prost::alloc::string::String>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ScannerActivityRequest {
-5
View File
@@ -817,11 +817,6 @@ message SignalServiceResponse {
bool success = 1;
optional string error_info = 2;
uint32 protocol_version = 3;
// Redacted fingerprint of the KMS configuration this node is running.
// Only set for the "kms" dynamic config subsystem, and left unset when the
// responder has no KMS configuration. Peers built before that subsystem
// reject the signal outright rather than answering without this field.
optional string config_fingerprint = 4;
}
message ScannerActivityRequest {
+4 -47
View File
@@ -241,16 +241,6 @@ impl InQueueStats {
Self::default()
}
pub fn add_current(&self, bytes: i64, count: i64) {
self.now_bytes.fetch_add(bytes.max(0), Ordering::Relaxed);
self.now_count.fetch_add(count.max(0), Ordering::Relaxed);
}
pub fn subtract_current(&self, bytes: i64, count: i64) {
saturating_atomic_sub(&self.now_bytes, bytes.max(0));
saturating_atomic_sub(&self.now_count, count.max(0));
}
pub fn get_current_bytes(&self) -> i64 {
self.now_bytes.load(Ordering::Relaxed)
}
@@ -260,14 +250,6 @@ impl InQueueStats {
}
}
fn saturating_atomic_sub(value: &AtomicI64, delta: i64) {
if delta == 0 {
return;
}
let _ = value.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(delta).max(0)));
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct InQueueMetric {
pub curr: InQueueStats,
@@ -377,18 +359,6 @@ impl QueueCache {
Self::default()
}
pub fn inc(&mut self, bucket: &str, size: i64) {
let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
stats.curr.add_current(size, 1);
self.sr_queue_stats.curr.add_current(size, 1);
}
pub fn dec(&mut self, bucket: &str, size: i64) {
let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
stats.curr.subtract_current(size, 1);
self.sr_queue_stats.curr.subtract_current(size, 1);
}
pub fn update(&mut self) {
let observed_at = Instant::now();
self.sr_queue_stats.observe(observed_at);
@@ -838,10 +808,12 @@ mod tests {
#[test]
fn in_queue_metric_observe_updates_rolling_stats() {
let mut metric = InQueueMetric::default();
metric.curr.add_current(128, 4);
metric.curr.now_bytes.store(128, Ordering::Relaxed);
metric.curr.now_count.store(4, Ordering::Relaxed);
metric.observe(Instant::now());
metric.curr.add_current(128, 2);
metric.curr.now_bytes.store(256, Ordering::Relaxed);
metric.curr.now_count.store(6, Ordering::Relaxed);
metric.observe(Instant::now());
assert_eq!(metric.curr.bytes, 256);
@@ -852,21 +824,6 @@ mod tests {
assert_eq!(metric.last_minute.count, 5);
}
#[test]
fn queue_cache_decrement_saturates_at_zero() {
let mut cache = QueueCache::new();
cache.inc("bucket", 128);
cache.dec("bucket", 512);
cache.dec("bucket", 1);
let bucket = cache.get_bucket_stats("bucket");
let site = cache.get_site_stats();
assert_eq!(bucket.curr.count, 0);
assert_eq!(bucket.curr.bytes, 0);
assert_eq!(site.curr.count, 0);
assert_eq!(site.curr.bytes, 0);
}
#[test]
fn fail_stats_recent_since_tracks_windows() {
let mut stats = FailStats::default();
-120
View File
@@ -86,20 +86,6 @@ pub enum EventName {
ObjectRemovedAbortMultipartUpload,
ObjectCreatedCreateMultipartUpload,
ObjectRemovedDeleteObjects,
// KMS management-plane events. They travel to the audit sink only and are
// never produced by the bucket notification path, so no compound `s3:`
// event expands to them. New variants must keep being appended here: the
// discriminant of every preceding variant is a `mask()` bit position, and
// inserting in the middle would silently renumber existing bits.
KmsKeyCreated,
KmsKeyRotated,
KmsKeyEnabled,
KmsKeyDisabled,
KmsKeyDeletionScheduled,
KmsKeyDeletionCancelled,
KmsKeyDeleted,
KmsKeyAccessed,
}
// Single event type sequential array for Everything.expand()
@@ -200,17 +186,6 @@ impl EventName {
"s3:Scanner:LargeVersions" => Ok(EventName::ScannerLargeVersions),
"s3:Scanner:BigPrefix" => Ok(EventName::ScannerBigPrefix),
"s3:Scanner:*" => Ok(EventName::ObjectScannerAll),
// KMS events use their own namespace so a `s3:` wildcard in a bucket
// notification config can never select them. They still round-trip
// because audit entries are persisted and replayed by the store targets.
"kms:Key:Created" => Ok(EventName::KmsKeyCreated),
"kms:Key:Rotated" => Ok(EventName::KmsKeyRotated),
"kms:Key:Enabled" => Ok(EventName::KmsKeyEnabled),
"kms:Key:Disabled" => Ok(EventName::KmsKeyDisabled),
"kms:Key:DeletionScheduled" => Ok(EventName::KmsKeyDeletionScheduled),
"kms:Key:DeletionCancelled" => Ok(EventName::KmsKeyDeletionCancelled),
"kms:Key:Deleted" => Ok(EventName::KmsKeyDeleted),
"kms:Key:Accessed" => Ok(EventName::KmsKeyAccessed),
// `Everything` has no string representation (`as_str` yields ""), so it
// cannot be parsed back from a string. Every other variant round-trips.
_ => Err(ParseEventNameError(s.to_string())),
@@ -276,14 +251,6 @@ impl EventName {
EventName::ObjectRemovedAbortMultipartUpload => "s3:ObjectRemoved:AbortMultipartUpload",
EventName::ObjectCreatedCreateMultipartUpload => "s3:ObjectCreated:CreateMultipartUpload",
EventName::ObjectRemovedDeleteObjects => "s3:ObjectRemoved:DeleteObjects",
EventName::KmsKeyCreated => "kms:Key:Created",
EventName::KmsKeyRotated => "kms:Key:Rotated",
EventName::KmsKeyEnabled => "kms:Key:Enabled",
EventName::KmsKeyDisabled => "kms:Key:Disabled",
EventName::KmsKeyDeletionScheduled => "kms:Key:DeletionScheduled",
EventName::KmsKeyDeletionCancelled => "kms:Key:DeletionCancelled",
EventName::KmsKeyDeleted => "kms:Key:Deleted",
EventName::KmsKeyAccessed => "kms:Key:Accessed",
}
}
@@ -390,25 +357,6 @@ impl EventName {
| EventName::ObjectRemovedDeleteObjects
)
}
/// Returns `true` for KMS management-plane events.
///
/// These are audit-only: they are never emitted through the bucket
/// notification pipeline, and no `s3:` event selector expands to them.
#[inline]
pub fn is_kms(&self) -> bool {
matches!(
self,
EventName::KmsKeyCreated
| EventName::KmsKeyRotated
| EventName::KmsKeyEnabled
| EventName::KmsKeyDisabled
| EventName::KmsKeyDeletionScheduled
| EventName::KmsKeyDeletionCancelled
| EventName::KmsKeyDeleted
| EventName::KmsKeyAccessed
)
}
}
/// Returns the S3 notification event schema version for a given event.
@@ -622,26 +570,6 @@ mod tests {
EventName::ObjectRemovedAbortMultipartUpload,
EventName::ObjectCreatedCreateMultipartUpload,
EventName::ObjectRemovedDeleteObjects,
EventName::KmsKeyCreated,
EventName::KmsKeyRotated,
EventName::KmsKeyEnabled,
EventName::KmsKeyDisabled,
EventName::KmsKeyDeletionScheduled,
EventName::KmsKeyDeletionCancelled,
EventName::KmsKeyDeleted,
EventName::KmsKeyAccessed,
];
/// Every KMS management-plane event.
const KMS_EVENT_NAMES: &[EventName] = &[
EventName::KmsKeyCreated,
EventName::KmsKeyRotated,
EventName::KmsKeyEnabled,
EventName::KmsKeyDisabled,
EventName::KmsKeyDeletionScheduled,
EventName::KmsKeyDeletionCancelled,
EventName::KmsKeyDeleted,
EventName::KmsKeyAccessed,
];
/// Regression for backlog#965: `mask()` used to recurse forever for the
@@ -754,54 +682,6 @@ mod tests {
}
}
/// KMS events are audit-plane only. No `s3:` selector — including the
/// catch-all `Everything` and every compound "All" type — may share a bit
/// with them, otherwise a bucket notification rule would silently start
/// matching KMS activity.
#[test]
fn test_kms_event_masks_are_disjoint_from_every_s3_selector() {
let s3_selectors: Vec<EventName> = ALL_EVENT_NAMES.iter().copied().filter(|ev| !ev.is_kms()).collect();
let mut seen = 0u64;
for kms in KMS_EVENT_NAMES {
let mask = kms.mask();
assert_ne!(mask, 0, "KMS event {kms} must have a non-zero mask");
assert_eq!(seen & mask, 0, "KMS event {kms} mask overlaps another KMS event");
seen |= mask;
for s3 in &s3_selectors {
assert_eq!(s3.mask() & mask, 0, "KMS event {kms} mask collides with S3 selector {s3}");
}
}
}
/// KMS event names must live in their own namespace so that neither a
/// `s3:` prefix filter nor an `s3:...:*` wildcard can select them.
#[test]
fn test_kms_event_names_are_outside_the_s3_namespace() {
for ev in KMS_EVENT_NAMES {
assert!(ev.is_kms(), "{ev} should be classified as a KMS event");
assert!(ev.as_str().starts_with("kms:"), "unexpected KMS event name {:?}", ev.as_str());
assert_eq!(EventName::parse(ev.as_str()).as_ref(), Ok(ev), "KMS event {ev} must round-trip");
assert_eq!(ev.expand(), vec![*ev], "KMS event {ev} must expand to itself only");
}
for ev in ALL_EVENT_NAMES.iter().filter(|ev| !ev.is_kms()) {
assert!(!ev.as_str().starts_with("kms:"), "{ev} must not claim the KMS namespace");
}
}
/// `mask()` shifts by `discriminant - 1`, so the enum may hold at most 64
/// mask-bearing variants. Appending past that overflows the shift instead
/// of failing loudly, so keep the budget check next to the variants.
#[test]
fn test_mask_bit_budget_is_not_exhausted() {
for ev in ALL_EVENT_NAMES {
let value = *ev as u32;
assert!(value <= 64, "{ev} has discriminant {value}; mask() only has 64 bits to hand out");
}
}
/// `Everything` must cover every sequential single-type bit.
#[test]
fn test_everything_mask_covers_all_single_types() {
+68 -99
View File
@@ -75,7 +75,7 @@ const DATA_SCANNER_COMPACT_LEAST_OBJECT: usize = 500;
const DATA_SCANNER_COMPACT_AT_CHILDREN: usize = 10000;
const DATA_SCANNER_COMPACT_AT_FOLDERS: usize = DATA_SCANNER_COMPACT_AT_CHILDREN / 4;
const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: usize = 250_000;
const SCANNER_LIST_PATH_RAW_STALL_TIMEOUT: Duration = Duration::from_secs(60);
const SCANNER_LIST_PATH_RAW_TIMEOUT: Duration = Duration::from_secs(60);
const SCANNER_ENTRY_PROGRESS_BATCH: u64 = 32;
const SCANNER_ENTRY_PROGRESS_INTERVAL: Duration = Duration::from_secs(30);
const DEFAULT_HEAL_OBJECT_SELECT_PROB: u32 = 1024;
@@ -100,21 +100,6 @@ static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new();
static SCANNER_INLINE_HEAL_METRICS_ONCE: Once = Once::new();
static SCANNER_ALERT_METRICS_ONCE: Once = Once::new();
#[cfg(test)]
type ListPathRawTimeoutSnapshot = (bool, Option<Duration>, Option<Duration>);
fn scanner_abandoned_child_list_options() -> ListPathRawOptions {
// A complete heal walk scales with bucket size and may legitimately take
// longer than a fixed wall-clock budget. Keep the total duration unbounded;
// Retain the scanner's per-read stall budget and keep cancellation controlled
// by the scanner cycle token.
ListPathRawOptions {
skip_walkdir_total_timeout: true,
walkdir_stall_timeout: Some(SCANNER_LIST_PATH_RAW_STALL_TIMEOUT),
..Default::default()
}
}
pub fn data_usage_update_dir_cycles() -> u32 {
rustfs_utils::get_env_u32(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, DATA_USAGE_UPDATE_DIR_CYCLES)
}
@@ -1296,8 +1281,6 @@ pub struct FolderScanner {
skip_heal: Arc<std::sync::atomic::AtomicBool>,
local_disk: Arc<Disk>,
pending_heals_changed: bool,
#[cfg(test)]
list_path_raw_options_observer: Option<mpsc::UnboundedSender<ListPathRawTimeoutSnapshot>>,
}
impl FolderScanner {
@@ -2427,79 +2410,75 @@ impl FolderScanner {
let bucket_clone = bucket.clone();
let prefix_clone = prefix.clone();
let child_ctx_clone = child_ctx.clone();
#[cfg(test)]
let list_path_raw_options_observer = self.list_path_raw_options_observer.clone();
tokio::spawn(async move {
let options = ListPathRawOptions {
disks,
bucket: bucket_clone.clone(),
path: prefix_clone.clone(),
recursive: true,
report_not_found: true,
min_disks: disks_quorum,
agreed: Some(Box::new(move |entry: MetaCacheEntry| {
let entry_name = entry.name.clone();
let agreed_tx = agreed_tx.clone();
Box::pin(async move {
if let Err(e) = agreed_tx.send(entry_name).await {
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
entry = %entry.name,
state = "list_path_agreed_send_failed",
error = %e,
"Scanner list_path_raw agreed callback failed"
);
}
})
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
let partial_tx = partial_tx.clone();
Box::pin(async move {
if let Err(e) = partial_tx.send(entries).await {
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
state = "list_path_partial_send_failed",
error = %e,
"Scanner list_path_raw partial callback failed"
);
}
})
})),
finished: Some(Box::new(move |errs: &[Option<DiskError>]| {
let finished_tx = finished_tx.clone();
let errs_clone = errs.to_vec();
Box::pin(async move {
if let Err(e) = finished_tx.send(errs_clone).await {
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
state = "list_path_finished_send_failed",
error = %e,
"Scanner list_path_raw finished callback failed"
);
}
})
})),
..scanner_abandoned_child_list_options()
};
#[cfg(test)]
if let Some(observer) = list_path_raw_options_observer {
let _ = observer.send((
options.skip_walkdir_total_timeout,
options.walkdir_timeout,
options.walkdir_stall_timeout,
));
}
if let Err(e) = list_path_raw(child_ctx_clone.clone(), options).await {
if let Err(e) = list_path_raw(
child_ctx_clone.clone(),
ListPathRawOptions {
disks,
bucket: bucket_clone.clone(),
path: prefix_clone.clone(),
recursive: true,
report_not_found: true,
min_disks: disks_quorum,
walkdir_timeout: Some(SCANNER_LIST_PATH_RAW_TIMEOUT),
walkdir_stall_timeout: Some(SCANNER_LIST_PATH_RAW_TIMEOUT),
agreed: Some(Box::new(move |entry: MetaCacheEntry| {
let entry_name = entry.name.clone();
let agreed_tx = agreed_tx.clone();
Box::pin(async move {
if let Err(e) = agreed_tx.send(entry_name).await {
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
entry = %entry.name,
state = "list_path_agreed_send_failed",
error = %e,
"Scanner list_path_raw agreed callback failed"
);
}
})
})),
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
let partial_tx = partial_tx.clone();
Box::pin(async move {
if let Err(e) = partial_tx.send(entries).await {
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
state = "list_path_partial_send_failed",
error = %e,
"Scanner list_path_raw partial callback failed"
);
}
})
})),
finished: Some(Box::new(move |errs: &[Option<DiskError>]| {
let finished_tx = finished_tx.clone();
let errs_clone = errs.to_vec();
Box::pin(async move {
if let Err(e) = finished_tx.send(errs_clone).await {
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
state = "list_path_finished_send_failed",
error = %e,
"Scanner list_path_raw finished callback failed"
);
}
})
})),
..Default::default()
},
)
.await
{
if is_missing_path_disk_error(&e) {
debug!(
target: "rustfs::scanner::folder",
@@ -2841,8 +2820,6 @@ pub async fn scan_data_folder(
skip_heal,
local_disk,
pending_heals_changed: false,
#[cfg(test)]
list_path_raw_options_observer: None,
};
// Check if context is cancelled
@@ -3049,7 +3026,6 @@ mod tests {
skip_heal: Arc::new(AtomicBool::new(false)),
local_disk: disk,
pending_heals_changed: false,
list_path_raw_options_observer: None,
};
(scanner, temp_dir)
@@ -4234,8 +4210,6 @@ mod tests {
scanner.heal_object_select = 1;
scanner.disks = disks;
scanner.disks_quorum = 2;
let (options_tx, mut options_rx) = mpsc::unbounded_channel();
scanner.list_path_raw_options_observer = Some(options_tx);
scanner.old_cache.replace(
&format!("{bucket}/{object}"),
bucket,
@@ -4257,11 +4231,6 @@ mod tests {
.expect("scan_folder should not hang after list_path_raw finishes")
.expect("scan_folder should finish successfully");
let observed_options = tokio::time::timeout(Duration::from_secs(1), options_rx.recv())
.await
.expect("abandoned-child listing options should be observed promptly")
.expect("abandoned-child listing options channel should remain open");
assert_eq!(observed_options, (true, None, Some(SCANNER_LIST_PATH_RAW_STALL_TIMEOUT)));
let root = scanner
.new_cache
.checked_flatten(bucket)

Some files were not shown because too many files have changed in this diff Show More